00000800:00000100:1.1F:1790771220.008914:0:2387:0:(socklnd_cb.c:2294:ksocknal_find_timed_out_conn()) Timeout sending data to 12345-192.168.204.150@tcp (192.168.204.150:988) the network or that node may be down. 00000100:00100000:3.0:1790772129.704742:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772129.704771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772129.896760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772130.024788:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772130.216747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772130.408783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772130.664820:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772130.728774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772130.920770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772131.048786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772131.240759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772131.432890:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772131.688791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772131.752759:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772131.752898:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772131.944779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772132.072764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772132.264773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772132.456804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772132.712866:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772132.776793:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772132.776987:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772132.968791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772133.032765:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772133.032777:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772133.032786:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772133.032809:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1128 00000100:00080000:3.0:1790772133.032828:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772133.032849:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772133.032916:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534382464/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772133.033006:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772133.033015:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772133.033020:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772133.033030:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1128 00000100:00080000:3.0:1790772133.033041:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772133.033054:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772133.033073:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772133.033109:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534382592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772133.033126:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772133.033132:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772133.033136:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772133.033143:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1128 00000100:00080000:3.0:1790772133.033168:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772133.033176:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772133.033182:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772133.033207:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534382720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772133.033223:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772133.033229:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772133.033232:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772133.033237:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1128 00000100:00080000:3.0:1790772133.033245:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772133.033253:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772133.033262:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772133.033284:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534382848/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772133.033304:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772133.033313:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772133.033321:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772133.033336:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1128 00000100:00080000:3.0:1790772133.033345:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772133.033354:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772133.033365:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772133.033387:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534382976/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772133.033408:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772133.033418:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772133.033428:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772133.033443:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1128 00000100:00080000:3.0:1790772133.033455:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772133.033494:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772133.033512:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772133.033546:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534383104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772133.033606:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772133.033623:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772133.033626:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772133.033635:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1128 00000100:00080000:3.0:1790772133.033643:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772133.033696:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772133.033708:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772133.033737:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534383232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772133.033779:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772133.033790:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772133.033798:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772133.033816:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1128 00000100:00080000:3.0:1790772133.033831:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772133.033842:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772133.033852:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772133.033874:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534383360/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772133.033890:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772133.033896:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772133.033900:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772133.033912:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1128 00000100:00080000:3.0:1790772133.033924:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772133.033964:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772133.033977:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772133.034011:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534383488/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772133.096831:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772133.288785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772133.480822:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772133.736773:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772133.800796:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772133.800845:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772133.992752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772134.120787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772134.312810:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772134.504752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772134.760811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772134.824845:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772135.016784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772135.144875:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772135.336760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772135.528783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772135.784790:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772135.848808:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772136.040761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772136.168794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772136.360794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772136.552809:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772136.808763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772136.872781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772137.064737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772137.192814:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772137.384750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772137.576744:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772137.832904:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772137.896815:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772138.088770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772138.152810:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772138.152820:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772138.152835:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772138.152869:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1134 00000100:00080000:3.0:1790772138.152887:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772138.152907:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772138.152967:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534383616/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772138.153005:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772138.153010:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772138.153012:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772138.153016:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1134 00000100:00080000:3.0:1790772138.153020:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772138.153027:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772138.153040:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772138.153058:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534383744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772138.153073:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772138.153081:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772138.153090:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772138.153134:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1134 00000100:00080000:3.0:1790772138.153138:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772138.153151:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772138.153159:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772138.153179:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534383872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772138.153204:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772138.153210:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772138.153212:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772138.153217:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1134 00000100:00080000:3.0:1790772138.153222:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772138.153234:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772138.153253:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772138.153268:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534384000/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772138.153300:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772138.153305:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772138.153310:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772138.153319:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1134 00000100:00080000:3.0:1790772138.153325:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772138.153335:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772138.153368:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772138.153397:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534384128/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772138.153431:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772138.153448:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772138.153456:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772138.153491:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1134 00000100:00080000:3.0:1790772138.153501:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772138.153520:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772138.153535:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772138.153566:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534384256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772138.153581:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772138.153586:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772138.153590:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772138.153598:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1134 00000100:00080000:3.0:1790772138.153604:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772138.153615:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772138.153627:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772138.153646:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534384384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772138.153661:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772138.153666:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772138.153670:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772138.153729:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1134 00000100:00080000:3.0:1790772138.153740:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772138.153752:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772138.153763:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772138.153782:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534384512/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772138.153862:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772138.153872:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772138.153886:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772138.153912:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1134 00000100:00080000:3.0:1790772138.153920:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772138.153932:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772138.153941:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772138.153967:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534384640/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772138.408796:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772138.600756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772138.856786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772138.920787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772138.920911:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772139.112791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772139.240921:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772139.432838:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772139.624874:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772139.880786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772139.944838:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772139.944923:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772140.136754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772140.264787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772140.456769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772140.648831:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772140.904777:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772140.968813:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772140.968905:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772141.160748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772141.288846:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772141.480835:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772141.672761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772141.928881:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772141.992877:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772142.184796:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772142.312861:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772142.504846:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772142.696780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772142.952833:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772143.016736:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772143.016818:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772143.208827:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772143.336750:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772143.528748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772143.720848:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772143.976849:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772144.040857:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772144.232833:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772144.360785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772144.552826:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772144.744793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772145.000814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772145.064803:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772145.065120:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772145.256816:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772145.384799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772145.576735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772145.768732:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772146.024811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772146.088829:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772146.088898:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772146.280796:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772146.408865:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772146.600781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772146.792832:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772147.048812:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772147.112758:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772147.112857:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772147.304797:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772147.432813:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772147.624775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772147.816802:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772148.072833:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772148.136845:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772148.328751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772148.456801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772148.648797:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772148.840781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772149.097030:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772149.160769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772149.160966:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772149.352709:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772149.480773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772149.672783:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772149.864798:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772150.120796:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772150.184741:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772150.184813:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772150.376831:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772150.504845:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772150.696804:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772150.888831:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772151.144973:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772151.208828:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772151.400743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772151.528797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772151.720837:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772151.912821:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772152.169053:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772152.232811:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772152.232908:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772152.424803:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772152.552775:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772152.744709:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772152.936791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772153.192725:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772153.256757:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772153.256802:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772153.448743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772153.576847:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772153.768760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772153.960764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772154.216879:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772154.280830:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772154.472772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772154.600832:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772154.792848:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772154.984795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772155.240828:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772155.304874:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772155.496791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772155.624827:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772155.816773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772156.008787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772156.264840:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772156.328795:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772156.328887:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772156.520769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772156.648770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772156.840823:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772157.032845:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772157.288776:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772157.352874:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772157.544771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772157.672779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772157.864771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772158.056788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772158.312811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772158.376862:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772158.568784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772158.696873:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772158.888790:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772159.080793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772159.336814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772159.400775:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772159.400847:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772159.592792:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772159.720893:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772159.912803:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772160.104800:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772160.360817:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772160.424831:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772160.424910:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772160.616773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772160.744805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772160.936755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772161.128805:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772161.384793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772161.448803:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772161.640761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772161.768815:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772161.960783:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772162.152752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772162.408833:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772162.472835:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772162.664752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772162.792910:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772162.984746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772163.176767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772163.432816:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772163.496754:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772163.496844:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772163.688767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772163.816813:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772164.008773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772164.200757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772164.456844:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772164.520860:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772164.712752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772164.840805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772165.032793:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772165.224776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772165.480830:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772165.544790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772165.544888:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772165.736757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772165.864808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772166.056782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772166.248794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772166.504822:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772166.568885:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772166.760769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772166.888824:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772167.080763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772167.272795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772167.528836:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772167.592767:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772167.784754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772167.912833:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772168.104781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772168.296783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772168.552767:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772168.616824:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772168.616916:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772168.808764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772168.936842:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772169.128752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772169.320768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772169.576806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772169.832733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772169.960841:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772170.152823:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772170.344860:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772170.600861:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772170.664912:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772170.856764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772170.984798:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772171.176795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772171.368794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772171.624835:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772171.688775:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772171.880755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772172.008846:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772172.200801:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772172.392902:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772172.648843:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772172.713107:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772172.713472:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772172.904776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772173.032791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772173.224785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772173.416813:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772173.672824:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772173.736827:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772173.736887:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772173.928749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772174.056793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772174.248783:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772174.440808:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772174.696837:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772174.760847:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772174.952754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772175.080811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772175.272789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772175.464786:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772175.720786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772175.784795:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772175.784894:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772175.976774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772176.104839:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772176.296818:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772176.488796:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772176.744791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772176.808823:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772177.000795:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772177.128795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772177.320764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772177.512780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772177.768870:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772177.832770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772177.832845:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772178.024747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772178.152784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772178.344767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772178.536771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772178.792830:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772178.856886:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772179.048762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772179.112873:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772179.112892:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772179.112902:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772179.112924:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1175 00000100:00080000:3.0:1790772179.112945:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772179.112969:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772179.113090:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534392832/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772179.113149:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772179.113157:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772179.113160:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772179.113168:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1175 00000100:00080000:3.0:1790772179.113177:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772179.113192:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772179.113210:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772179.113254:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534392960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772179.113285:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772179.113293:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772179.113298:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772179.113310:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1175 00000100:00080000:3.0:1790772179.113322:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772179.113342:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772179.113355:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772179.113382:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534393088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772179.113398:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772179.113407:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772179.113417:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772179.113443:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1175 00000100:00080000:3.0:1790772179.113459:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772179.113473:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772179.113481:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772179.113501:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534393216/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772179.113583:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772179.113611:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772179.113617:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772179.113625:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1175 00000100:00080000:3.0:1790772179.113633:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772179.113640:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772179.113721:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772179.113760:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534393344/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772179.113784:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772179.113814:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772179.113820:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772179.113838:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1175 00000100:00080000:3.0:1790772179.113863:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772179.113882:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772179.113907:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772179.113949:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534393472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772179.113992:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772179.114010:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772179.114021:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772179.114064:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1175 00000100:00080000:3.0:1790772179.114093:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772179.114116:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772179.114144:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772179.114189:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534393600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772179.114215:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772179.114228:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772179.114239:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772179.114267:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1175 00000100:00080000:3.0:1790772179.114286:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772179.114313:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772179.114327:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772179.114357:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534393728/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772179.114405:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772179.114412:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772179.114418:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772179.114440:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1175 00000100:00080000:3.0:1790772179.114450:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772179.114464:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772179.114482:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772179.114525:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534393856/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772179.176799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772179.368789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772179.560756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772179.816779:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772179.880813:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772179.880927:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772180.072731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772180.200825:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772180.392822:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772180.584771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772180.840804:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772180.904826:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772181.096869:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772181.224788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772181.224860:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772181.416752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772181.608793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772181.864765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772181.928820:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772181.928867:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772182.120769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772182.248826:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772182.440781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772182.632784:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772182.888811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772182.952811:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772183.144739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772183.272793:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772183.464790:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772183.656790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772183.912801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772183.976819:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772184.168797:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772184.296816:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772184.488811:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772184.680781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772184.936750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772185.192754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772185.320786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772185.512775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772185.704787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772185.960782:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772186.025133:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772186.216761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772186.344834:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772186.536797:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772186.728820:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772186.984816:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772187.048782:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772187.048871:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772187.240765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772187.368772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772187.560805:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772187.752780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772188.008720:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772188.072755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772188.072814:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772188.264785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772188.392812:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772188.584814:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772188.776779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772189.032798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772189.096827:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772189.096937:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772189.288754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772189.416774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772189.608751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772189.800803:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772190.056863:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772190.120871:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772190.120954:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772190.312803:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772190.440861:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772190.632844:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772190.824963:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772191.080783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772191.144873:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772191.336759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772191.464831:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772191.656839:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772191.848791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772192.104920:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772192.168761:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772192.168857:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772192.360781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000400:00000100:3.0:1790772192.425250:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:3.0:1790772192.425327:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1790772192.425357:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790772192.425480:0:2383:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:3.0:1790772192.425507:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790772192.425525:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790772192.425545:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790772192.425564:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790772192.425587:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790772192.425608:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790772192.425632:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:3.0:1790772192.488811:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772192.680756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772192.872827:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772193.128844:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772193.193617:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772193.384791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772193.512791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772193.704831:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772193.896804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772194.152786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772194.408801:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772194.536876:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772194.728810:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772194.920846:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772195.176828:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772195.240895:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772195.432773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772195.560824:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772195.752793:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772195.944814:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772196.200813:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772196.264789:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772196.264920:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772196.456815:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772196.776757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772196.968838:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772197.224806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772197.288821:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772197.288906:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772197.480803:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772197.608821:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772197.800781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772197.992796:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772198.248750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772198.312762:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772198.504777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772198.632868:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772198.824798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772199.016812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772199.272800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772199.336815:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772199.528772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772199.592879:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772199.592890:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772199.592897:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772199.592914:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1195 00000100:00080000:3.0:1790772199.592933:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772199.592953:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772199.593022:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534397440/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772199.593067:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772199.593073:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772199.593076:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772199.593081:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1195 00000100:00080000:3.0:1790772199.593086:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772199.593092:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772199.593107:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772199.593125:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534397568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772199.593152:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772199.593157:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772199.593161:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772199.593167:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1195 00000100:00080000:3.0:1790772199.593171:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772199.593177:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772199.593184:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772199.593207:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534397696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772199.593234:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772199.593242:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772199.593246:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772199.593256:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1195 00000100:00080000:3.0:1790772199.593264:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772199.593294:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772199.593308:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772199.593345:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534397824/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772199.593363:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772199.593368:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772199.593372:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772199.593381:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1195 00000100:00080000:3.0:1790772199.593390:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772199.593406:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772199.593423:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772199.593447:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534397952/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772199.593473:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772199.593483:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772199.593491:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772199.593503:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1195 00000100:00080000:3.0:1790772199.593511:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772199.593523:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772199.593542:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772199.593563:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534398080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772199.593602:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772199.593609:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772199.593614:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772199.593633:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1195 00000100:00080000:3.0:1790772199.593709:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772199.593726:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772199.593743:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772199.593771:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534398208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772199.593819:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772199.593824:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772199.593828:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772199.593846:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1195 00000100:00080000:3.0:1790772199.593851:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772199.593859:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772199.593871:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772199.593898:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534398336/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772199.593919:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772199.593926:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772199.593930:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772199.593941:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1195 00000100:00080000:3.0:1790772199.593947:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772199.593962:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772199.593975:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772199.593996:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534398464/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772199.656819:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772199.848798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772200.040772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772200.296799:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772200.360786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772200.360886:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772200.552771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772200.680793:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772200.872785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772201.064775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772201.320866:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772201.385732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772201.576753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772201.704768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772201.896742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772202.088812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772202.344799:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772202.409233:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772202.600724:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772202.728771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772202.920724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772203.112699:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772203.368765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772203.432795:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772203.624799:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772203.752791:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772203.944770:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772204.136780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772204.392834:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772204.456786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772204.456916:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772204.648752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772204.712775:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772204.712788:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772204.712796:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772204.712818:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1200 00000100:00080000:3.0:1790772204.712838:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772204.712861:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772204.712929:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534398592/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772204.712975:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772204.712980:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772204.712983:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772204.712990:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1200 00000100:00080000:3.0:1790772204.712996:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772204.713003:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772204.713020:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772204.713047:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534398720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772204.713084:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772204.713089:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772204.713093:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772204.713101:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1200 00000100:00080000:3.0:1790772204.713124:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772204.713133:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772204.713143:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772204.713161:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534398848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772204.713175:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772204.713192:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772204.713203:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772204.713212:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1200 00000100:00080000:3.0:1790772204.713218:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772204.713228:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772204.713236:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772204.713259:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534398976/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772204.713277:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772204.713302:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772204.713318:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772204.713340:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1200 00000100:00080000:3.0:1790772204.713359:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772204.713366:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772204.713375:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772204.713396:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534399104/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772204.713452:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772204.713471:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772204.713474:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772204.713492:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1200 00000100:00080000:3.0:1790772204.713503:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772204.713523:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772204.713533:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772204.713589:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534399232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772204.713603:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772204.713608:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772204.713611:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772204.713617:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1200 00000100:00080000:3.0:1790772204.713621:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772204.713627:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772204.713643:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772204.713693:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534399360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772204.713706:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772204.713711:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772204.713714:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772204.713719:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1200 00000100:00080000:3.0:1790772204.713724:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772204.713730:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772204.713735:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772204.713758:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534399488/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772204.713771:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772204.713776:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772204.713779:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772204.713785:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1200 00000100:00080000:3.0:1790772204.713793:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772204.713803:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772204.713812:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772204.713829:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534399616/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772204.776811:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772204.968787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772205.160794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772205.416825:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772205.480812:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772205.672766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772205.800952:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772205.992799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772206.184800:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772206.440787:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772206.504845:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772206.696797:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772206.824793:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772207.016759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772207.208771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772207.464812:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772207.528793:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772207.720754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772207.848777:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772208.040778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772208.232816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772208.488746:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772208.552762:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772208.552836:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772208.744785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772208.872789:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772209.064762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772209.256700:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772209.512764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772209.576850:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772209.768764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772209.832789:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772209.832810:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772209.832821:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772209.832841:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1205 00000100:00080000:3.0:1790772209.832862:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772209.832884:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772209.832968:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534399744/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772209.833040:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772209.833046:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772209.833051:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772209.833059:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1205 00000100:00080000:3.0:1790772209.833066:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772209.833076:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772209.833092:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772209.833129:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534399872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772209.833158:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772209.833167:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772209.833173:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772209.833190:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1205 00000100:00080000:3.0:1790772209.833225:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772209.833240:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772209.833249:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772209.833295:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534400000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772209.833355:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772209.833371:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772209.833381:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772209.833393:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1205 00000100:00080000:3.0:1790772209.833400:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772209.833411:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772209.833426:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772209.833499:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534400128/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772209.833562:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772209.833567:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772209.833570:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772209.833578:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1205 00000100:00080000:3.0:1790772209.833583:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772209.833594:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772209.833608:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772209.833634:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534400256/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772209.833727:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772209.833743:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772209.833756:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772209.833788:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1205 00000100:00080000:3.0:1790772209.833810:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772209.833826:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772209.833840:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772209.833876:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534400384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772209.833906:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772209.833932:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772209.833939:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772209.833954:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1205 00000100:00080000:3.0:1790772209.833976:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772209.834011:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772209.834031:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772209.834062:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534400512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772209.834078:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772209.834089:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772209.834094:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772209.834104:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1205 00000100:00080000:3.0:1790772209.834121:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772209.834144:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772209.834163:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772209.834214:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534400640/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772209.834237:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772209.834249:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772209.834287:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772209.834300:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1205 00000100:00080000:3.0:1790772209.834314:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772209.834335:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772209.834351:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772209.834387:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534400768/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772209.896811:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772210.088794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772210.280715:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772210.536844:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772210.600832:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772210.600951:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772210.792806:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772210.920842:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772211.112819:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772211.304790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772211.560859:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772211.624846:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772211.816758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772211.944847:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772212.136781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772212.328763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772212.456850:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772212.584815:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772212.648777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772212.648923:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772212.840711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772212.968725:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772213.160736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772213.352760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772213.480754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772213.608790:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772213.672809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772213.864781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772213.992765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772214.184804:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772214.376792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772214.504808:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772214.632813:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772214.696853:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772214.888797:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772214.953639:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772214.953688:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1210 00000100:00080000:0.0:1790772214.953694:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772214.953704:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772214.953714:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772214.953738:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534401920/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772215.016776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772215.016846:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772215.080749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772215.080959:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772215.208818:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772215.400854:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772215.464785:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772215.528776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772215.656926:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772215.720737:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772215.720805:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772215.720829:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772215.720892:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772215.848769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772215.912742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772216.040822:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772216.040823:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772216.104900:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772216.232810:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772216.424788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772216.488723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772216.552760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772216.680902:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772216.744840:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772216.744854:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772216.872773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772216.936781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772217.064826:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772217.064843:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772217.128814:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772217.128909:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772217.256792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772217.448844:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772217.512774:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772217.576864:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772217.704814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772217.704922:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772217.768805:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772217.768921:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772217.768933:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772217.769012:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772217.896739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772217.960753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772218.088809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772218.088858:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772218.152786:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772218.152840:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772218.152959:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772218.280765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772218.472795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772218.536808:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772218.600829:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772218.728899:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772218.728911:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772218.792849:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772218.792956:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772218.793145:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772218.920765:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772218.984750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772219.112828:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772219.112875:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772219.176799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772219.304788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772219.496780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772219.560777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772219.624823:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772219.752799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772219.753768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772219.816831:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772219.816875:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772219.944794:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772220.008779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772220.136798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772220.136814:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772220.200876:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772220.328750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772220.520819:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772220.584754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772220.648811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772220.776824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772220.777718:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772220.840801:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772220.840855:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772220.840938:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772220.968744:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772221.032779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772221.160865:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772221.160881:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772221.224842:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772221.352808:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772221.544726:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772221.608736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772221.672811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772221.800773:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772221.800796:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772221.864893:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772221.864902:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772221.864957:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772221.992724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772222.056751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772222.184762:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772222.184762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772222.248794:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772222.376783:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772222.568813:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772222.632765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772222.696818:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772222.824813:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772222.824813:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772222.888839:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772222.888892:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772222.889792:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772222.889849:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772223.016759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772223.080785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772223.208820:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772223.208820:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772223.272779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772223.400780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772223.592788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772223.656786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772223.720818:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772223.848779:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772223.848852:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772223.912795:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772223.912845:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772223.912905:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772224.040736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772224.104785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772224.232786:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772224.232815:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772224.296847:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772224.424802:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772224.616827:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772224.680737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772224.744809:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772224.872796:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772224.872843:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772224.936834:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772224.936880:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772224.936966:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772225.064749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772225.128766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772225.192843:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772225.192854:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772225.192861:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772225.192877:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1221 00000100:00080000:3.0:1790772225.192895:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772225.192915:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772225.193000:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534403200/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772225.193044:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772225.193050:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772225.193053:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772225.193059:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1221 00000100:00080000:3.0:1790772225.193064:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772225.193072:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772225.193100:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772225.193127:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534403328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772225.193177:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772225.193182:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772225.193185:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772225.193193:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1221 00000100:00080000:3.0:1790772225.193198:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772225.193205:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772225.193215:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772225.193234:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534403456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772225.193248:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772225.193254:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772225.193257:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772225.193262:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1221 00000100:00080000:3.0:1790772225.193268:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772225.193280:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772225.193294:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772225.193311:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534403584/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772225.193324:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772225.193329:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772225.193332:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772225.193339:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1221 00000100:00080000:3.0:1790772225.193344:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772225.193352:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772225.193359:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772225.193379:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534403712/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772225.193395:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772225.193402:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772225.193411:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772225.193428:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1221 00000100:00080000:3.0:1790772225.193438:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772225.193452:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772225.193458:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772225.193473:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534403840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772225.193502:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772225.193509:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772225.193522:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772225.193531:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1221 00000100:00080000:3.0:1790772225.193538:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772225.193551:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772225.193561:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772225.193582:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534403968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772225.193601:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772225.193608:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772225.193612:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772225.193622:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1221 00000100:00080000:3.0:1790772225.193631:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772225.193702:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772225.193720:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772225.193749:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534404096/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772225.193780:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772225.193787:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772225.193793:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772225.193810:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1221 00000100:00080000:3.0:1790772225.193815:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772225.193829:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772225.193844:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772225.193871:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534404224/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772225.256878:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772225.256894:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772225.320842:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772225.448811:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772225.640842:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772225.704806:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772225.768831:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772225.896762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772225.896836:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772225.960882:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772225.960986:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772225.961175:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772226.088782:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772226.152754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772226.280739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772226.280753:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772226.344921:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772226.472820:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772226.664781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772226.664938:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772226.728761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772226.792778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772226.920790:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772226.920810:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772226.984895:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772226.985087:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772226.985205:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772227.112795:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772227.176765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772227.304795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772227.304834:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772227.496765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772227.688804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772227.688868:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772227.752790:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772227.816792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772227.944877:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772227.944897:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772228.008812:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772228.008896:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772228.008901:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772228.136744:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772228.200841:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772228.328762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772228.328780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772228.520756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772228.712742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772228.712797:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772228.776751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772228.840762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772228.968784:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772228.968798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772229.032747:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772229.032774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772229.032894:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772229.160732:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772229.224775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772229.352776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772229.352800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772229.416872:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772229.544767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772229.736825:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772229.736838:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772229.800767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772229.864772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772229.992802:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772229.992820:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772230.056888:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772230.056942:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772230.057490:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772230.184747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772230.248794:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772230.312830:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772230.376880:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772230.376886:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772230.440859:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772230.568827:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772230.760816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772230.760843:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772230.824730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772230.888748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772231.016746:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772231.016852:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772231.080798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772231.080840:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772231.080925:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772231.208771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772231.272788:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772231.336764:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772231.400813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772231.400836:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772231.464813:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772231.592805:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772231.784735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772231.784743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772231.848749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772231.912787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772232.040853:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772232.040855:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772232.104797:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772232.104824:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772232.232752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772232.296805:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772232.360773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772232.424759:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772232.424810:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772232.488806:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772232.616775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772232.808791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772232.808816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772232.872743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772232.936709:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772233.064726:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772233.064727:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772233.128721:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772233.128755:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772233.128768:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772233.256752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772233.320784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772233.384760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772233.448775:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772233.448804:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772233.640782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772233.832769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772233.832824:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772233.896767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772233.960751:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772234.088767:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772234.088770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772234.152801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772234.152814:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772234.280794:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772234.344775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772234.408781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772234.472808:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772234.472809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772234.664794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772234.856791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772234.856794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772234.920783:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772234.984844:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772235.112827:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772235.112839:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772235.176739:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772235.176843:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772235.176936:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772235.304794:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772235.368805:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772235.432806:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772235.432830:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772235.432839:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772235.432858:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1231 00000100:00080000:3.0:1790772235.432876:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772235.432897:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772235.433009:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534405504/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772235.433056:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772235.433062:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772235.433065:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772235.433071:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1231 00000100:00080000:3.0:1790772235.433077:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772235.433085:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772235.433103:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772235.433137:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534405632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772235.433167:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772235.433190:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772235.433194:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772235.433208:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1231 00000100:00080000:3.0:1790772235.433217:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772235.433227:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772235.433239:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772235.433264:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534405760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772235.433284:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772235.433301:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772235.433306:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772235.433320:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1231 00000100:00080000:3.0:1790772235.433329:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772235.433344:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772235.433358:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772235.433418:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534405888/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772235.433541:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772235.433554:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772235.433563:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772235.433581:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1231 00000100:00080000:3.0:1790772235.433596:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772235.433617:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772235.433633:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772235.433710:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534406016/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772235.433732:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772235.433747:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772235.433753:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772235.433764:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1231 00000100:00080000:3.0:1790772235.433774:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772235.433792:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772235.433816:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772235.433843:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534406144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772235.433868:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772235.433875:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772235.433885:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772235.433903:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1231 00000100:00080000:3.0:1790772235.433916:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772235.433933:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772235.433978:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772235.434017:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534406272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772235.434036:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772235.434052:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772235.434056:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772235.434066:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1231 00000100:00080000:3.0:1790772235.434078:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772235.434098:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772235.434112:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772235.434162:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534406400/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772235.434228:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772235.434234:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772235.434239:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772235.434259:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1231 00000100:00080000:3.0:1790772235.434282:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772235.434300:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772235.434322:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772235.434370:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534406528/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772235.496896:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772235.496899:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772235.688805:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772235.880776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772235.880819:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772235.944709:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772236.008773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772236.136856:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772236.136865:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772236.200811:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772236.200836:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772236.200909:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772236.328873:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772236.392788:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772236.520776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772236.520789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772236.584845:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772236.712851:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772236.904767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772236.904820:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772236.968756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772237.032797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772237.160786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772237.160836:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772237.224800:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772237.224814:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772237.224936:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772237.352773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772237.416766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772237.544816:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772237.544858:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772237.608820:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772237.608981:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772237.736783:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772237.928745:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772237.928793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772237.992804:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772238.056872:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772238.184837:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772238.184839:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772238.248825:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772238.248841:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772238.248952:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772238.376793:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772238.440761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772238.568947:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772238.568965:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772238.632843:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772238.760766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772238.952769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772238.952834:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772239.016754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772239.080800:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772239.208782:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772239.208836:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772239.272762:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772239.272808:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772239.272862:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772239.272931:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772239.400796:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772239.464773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772239.592830:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772239.592831:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772239.656793:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772239.657007:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772239.784780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772239.976824:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772239.976825:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772240.040764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772240.104825:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772240.232784:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772240.232912:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772240.296877:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772240.296913:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772240.424778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772240.488753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772240.552774:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772240.552790:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772240.552798:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772240.552815:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1236 00000100:00080000:3.0:1790772240.552832:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772240.552878:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772240.552946:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534406656/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772240.552990:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772240.552996:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772240.552999:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772240.553005:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1236 00000100:00080000:3.0:1790772240.553011:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772240.553018:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772240.553036:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772240.553056:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534406784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772240.553071:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772240.553077:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772240.553082:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772240.553092:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1236 00000100:00080000:3.0:1790772240.553100:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772240.553118:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772240.553137:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772240.553177:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534406912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772240.553202:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772240.553209:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772240.553213:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772240.553220:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1236 00000100:00080000:3.0:1790772240.553231:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772240.553252:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772240.553264:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772240.553284:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534407040/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772240.553299:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772240.553304:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772240.553310:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772240.553324:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1236 00000100:00080000:3.0:1790772240.553333:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772240.553346:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772240.553359:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772240.553384:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534407168/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772240.553401:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772240.553419:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772240.553427:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772240.553446:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1236 00000100:00080000:3.0:1790772240.553458:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772240.553470:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772240.553481:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772240.553494:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534407296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772240.553508:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772240.553527:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772240.553533:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772240.553567:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1236 00000100:00080000:3.0:1790772240.553587:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772240.553608:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772240.553626:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772240.553719:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534407424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772240.553781:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772240.553788:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772240.553794:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772240.553822:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1236 00000100:00080000:3.0:1790772240.553839:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772240.553879:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772240.553892:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772240.553919:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534407552/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772240.553954:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772240.553979:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772240.553983:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772240.553991:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1236 00000100:00080000:3.0:1790772240.553999:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772240.554010:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772240.554017:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772240.554038:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534407680/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772240.616824:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772240.616833:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00010000:00010000:0.0:1790772240.681051:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772240.808756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772241.000769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772241.000779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772241.064771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772241.128824:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772241.256808:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772241.256816:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772241.320785:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772241.320801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772241.320938:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772241.320953:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772241.448766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772241.512746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772241.640784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772241.640799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00010000:00010000:0.0:1790772241.705076:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772241.832842:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772242.024775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772242.024811:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772242.088718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772242.152755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772242.280787:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772242.280862:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772242.344806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772242.344860:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772242.344915:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772242.472755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772242.536709:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772242.664757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772242.664757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00010000:00010000:0.0:1790772242.729095:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772242.856768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772243.048753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772243.048793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772243.112775:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772243.176845:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772243.304787:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772243.304827:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772243.368873:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772243.368986:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772243.369088:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772243.369695:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772243.496869:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772243.560840:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772243.688940:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772243.688941:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772243.752875:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772243.752989:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772243.880821:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772244.072860:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772244.072861:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772244.136812:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772244.200846:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772244.328753:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772244.328753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772244.392743:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772244.392751:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772244.392837:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772244.520750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772244.584736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772244.712754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772244.712818:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772244.776787:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772244.904826:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772245.096870:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772245.096906:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772245.160749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772245.224853:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772245.352798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772245.352839:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772245.416858:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772245.416920:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772245.416959:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772245.544802:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772245.608793:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772245.672837:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772245.672848:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772245.672856:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772245.672884:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1241 00000100:00080000:3.0:1790772245.672902:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772245.672922:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772245.673010:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534407808/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772245.673082:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772245.673089:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772245.673092:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772245.673100:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1241 00000100:00080000:3.0:1790772245.673107:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772245.673116:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772245.673134:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772245.673160:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534407936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772245.673183:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772245.673202:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772245.673209:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772245.673217:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1241 00000100:00080000:3.0:1790772245.673222:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772245.673230:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772245.673241:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772245.673257:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534408064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772245.673270:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772245.673275:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772245.673278:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772245.673283:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1241 00000100:00080000:3.0:1790772245.673287:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772245.673292:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772245.673298:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772245.673312:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534408192/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772245.673326:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772245.673331:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772245.673333:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772245.673339:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1241 00000100:00080000:3.0:1790772245.673346:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772245.673354:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772245.673362:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772245.673394:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534408320/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772245.673408:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772245.673414:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772245.673418:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772245.673424:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1241 00000100:00080000:3.0:1790772245.673429:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772245.673439:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772245.673448:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772245.673465:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534408448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772245.673479:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772245.673486:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772245.673491:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772245.673513:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1241 00000100:00080000:3.0:1790772245.673527:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772245.673538:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772245.673547:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772245.673572:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534408576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772245.673623:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772245.673706:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772245.673715:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772245.673730:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1241 00000100:00080000:3.0:1790772245.673741:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772245.673798:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772245.673817:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772245.673855:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534408704/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772245.673872:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772245.673877:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772245.673881:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772245.673897:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1241 00000100:00080000:3.0:1790772245.673907:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772245.673920:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772245.673928:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772245.673962:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534408832/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772245.736842:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772245.736856:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772245.800865:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772245.928835:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772246.120791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772246.120801:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772246.184794:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772246.248824:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772246.376816:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772246.376837:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772246.440894:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772246.441003:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772246.568795:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772246.632758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772246.760811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772246.760821:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772246.824808:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772246.825006:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772246.952855:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772247.144781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772247.144807:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772247.208816:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772247.272790:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772247.400826:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772247.400826:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772247.464822:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772247.464928:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772247.465698:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772247.592733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772247.656816:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772247.784852:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772247.784901:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772247.848824:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772247.848885:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772247.976800:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772248.168785:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772248.168796:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772248.232768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772248.296775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772248.424745:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772248.424745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772248.488713:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772248.488748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772248.488777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772248.488786:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772248.616740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772248.680790:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772248.808797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772248.808827:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772248.872836:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772249.000779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772249.192783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772249.192789:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772249.256777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772249.320824:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772249.448806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772249.448875:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772249.512838:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772249.512838:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772249.512946:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772249.640773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772249.704786:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772249.832824:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772249.832875:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772249.896785:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772250.024781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772250.216777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772250.216789:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772250.280788:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772250.344833:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772250.472799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772250.472846:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772250.536883:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772250.537321:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772250.537507:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772250.664770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772250.728796:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772250.792798:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772250.792815:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772250.792823:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772250.792839:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1246 00000100:00080000:3.0:1790772250.792856:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772250.792874:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772250.792938:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534408960/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772250.792993:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772250.792999:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772250.793002:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772250.793008:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1246 00000100:00080000:3.0:1790772250.793012:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772250.793018:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772250.793035:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772250.793055:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534409088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772250.793071:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772250.793078:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772250.793085:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772250.793096:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1246 00000100:00080000:3.0:1790772250.793101:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772250.793106:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772250.793114:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772250.793131:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534409216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772250.793145:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772250.793151:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772250.793154:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772250.793159:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1246 00000100:00080000:3.0:1790772250.793166:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772250.793173:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772250.793179:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772250.793200:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534409344/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772250.793215:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772250.793221:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772250.793225:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772250.793231:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1246 00000100:00080000:3.0:1790772250.793236:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772250.793242:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772250.793248:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772250.793263:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534409472/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772250.793278:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772250.793295:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772250.793302:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772250.793316:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1246 00000100:00080000:3.0:1790772250.793322:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772250.793336:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772250.793349:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772250.793376:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534409600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772250.793422:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772250.793428:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772250.793431:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772250.793439:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1246 00000100:00080000:3.0:1790772250.793445:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772250.793452:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772250.793459:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772250.793481:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534409728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772250.793497:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772250.793508:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772250.793514:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772250.793523:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1246 00000100:00080000:3.0:1790772250.793530:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772250.793541:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772250.793554:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772250.793575:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534409856/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772250.793594:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772250.793608:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772250.793613:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772250.793629:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1246 00000100:00080000:3.0:1790772250.793639:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772250.793721:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772250.793731:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772250.793757:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534409984/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772250.856775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772250.856779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772250.920791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772251.048759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772251.240763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772251.240859:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772251.304774:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772251.368782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772251.496847:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772251.496851:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772251.560965:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772251.561111:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772251.561192:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772251.688781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772251.752814:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772251.880839:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772251.880854:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772251.944762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772252.072785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772252.264787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772252.264800:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772252.328789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772252.392796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772252.520818:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772252.520824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772252.584747:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772252.584815:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772252.584873:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772252.584874:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772252.712756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772252.776762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772252.904761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772252.904852:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772252.968772:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772253.096775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772253.288815:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772253.288816:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772253.352741:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772253.416764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772253.544716:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772253.544751:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772253.608815:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772253.608815:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772253.736747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772253.800739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772253.928797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772253.928840:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772253.992749:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772254.120762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772254.312784:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772254.312791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772254.376802:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772254.440819:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772254.568785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772254.568801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772254.632791:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772254.632823:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772254.760756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772254.824781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772254.952724:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772254.952730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772255.016731:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772255.144723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772255.336722:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772255.336723:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772255.400703:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772255.464727:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772255.592717:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772255.592717:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772255.656717:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772255.656717:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772255.656768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772255.784748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772255.848761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772255.912771:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772255.912783:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772255.912790:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772255.912809:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1251 00000100:00080000:3.0:1790772255.912830:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772255.912852:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772255.912920:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534410112/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772255.912963:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772255.912969:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772255.912972:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772255.912978:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1251 00000100:00080000:3.0:1790772255.912982:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772255.912988:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772255.913006:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772255.913037:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534410240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772255.913076:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772255.913084:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772255.913090:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772255.913101:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1251 00000100:00080000:3.0:1790772255.913111:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772255.913132:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772255.913144:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772255.913194:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534410368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772255.913219:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772255.913228:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772255.913233:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772255.913243:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1251 00000100:00080000:3.0:1790772255.913250:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772255.913264:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772255.913277:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772255.913300:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534410496/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772255.913318:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772255.913324:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772255.913326:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772255.913339:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1251 00000100:00080000:3.0:1790772255.913347:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772255.913357:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772255.913365:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772255.913390:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534410624/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772255.913431:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772255.913441:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772255.913451:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772255.913466:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1251 00000100:00080000:3.0:1790772255.913476:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772255.913489:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772255.913500:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772255.913529:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534410752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772255.913593:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772255.913602:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772255.913607:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772255.913617:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1251 00000100:00080000:3.0:1790772255.913626:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772255.913636:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772255.913645:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772255.913714:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534410880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772255.913734:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772255.913742:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772255.913747:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772255.913762:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1251 00000100:00080000:3.0:1790772255.913776:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772255.913790:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772255.913802:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772255.913838:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534411008/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772255.913871:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772255.913879:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772255.913884:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772255.913892:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1251 00000100:00080000:3.0:1790772255.913900:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772255.913912:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772255.913924:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772255.913950:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534411136/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772255.976785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772255.976800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772256.040792:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772256.168752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772256.360761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772256.360780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772256.424756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772256.488809:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772256.616778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772256.616790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772256.680757:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772256.680813:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772256.680825:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772256.680869:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772256.808755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772256.872764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772257.000801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772257.000802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772257.064784:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772257.064869:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772257.192780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772257.384755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772257.384773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772257.448767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772257.512795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772257.640773:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772257.640800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772257.704805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772257.704814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772257.832742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772257.896765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772258.024787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772258.024809:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772258.088829:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772258.216783:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772258.408781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772258.408791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772258.472773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772258.536797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772258.536836:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772258.664761:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772258.664790:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772258.728798:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772258.728805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772258.856742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772258.920784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772259.048776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772259.048782:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772259.112767:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772259.240757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772259.432807:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772259.432808:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772259.496791:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772259.560773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772259.560805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772259.688780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772259.688793:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772259.752769:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772259.752799:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772259.752845:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772259.880744:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772259.944760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772260.072789:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772260.072797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772260.136762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772260.264768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772260.456767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772260.456800:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772260.520790:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772260.584775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772260.584797:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772260.712816:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772260.776753:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772260.776792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772260.776821:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772260.904765:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772260.968770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772261.032804:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772261.032820:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772261.032828:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772261.032846:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1256 00000100:00080000:3.0:1790772261.032865:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772261.032886:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772261.032946:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534411264/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772261.032989:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772261.032994:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772261.032997:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772261.033003:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1256 00000100:00080000:3.0:1790772261.033007:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772261.033014:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772261.033031:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772261.033050:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534411392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772261.033067:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772261.033078:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772261.033086:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772261.033100:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1256 00000100:00080000:3.0:1790772261.033107:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772261.033118:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772261.033129:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772261.033154:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534411520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772261.033176:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772261.033184:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772261.033189:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772261.033198:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1256 00000100:00080000:3.0:1790772261.033206:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772261.033218:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772261.033230:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772261.033265:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534411648/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772261.033288:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772261.033295:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772261.033299:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772261.033308:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1256 00000100:00080000:3.0:1790772261.033316:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772261.033326:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772261.033337:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772261.033358:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534411776/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772261.033376:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772261.033383:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772261.033387:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772261.033395:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1256 00000100:00080000:3.0:1790772261.033404:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772261.033415:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772261.033438:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772261.033492:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534411904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772261.033514:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772261.033522:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772261.033527:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772261.033538:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1256 00000100:00080000:3.0:1790772261.033556:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772261.033567:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772261.033578:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772261.033603:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534412032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772261.033624:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772261.033632:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772261.033635:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772261.033645:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1256 00000100:00080000:3.0:1790772261.033697:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772261.033710:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772261.033721:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772261.033746:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534412160/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772261.033791:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772261.033799:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772261.033803:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772261.033811:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1256 00000100:00080000:3.0:1790772261.033818:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772261.033829:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772261.033839:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772261.033862:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534412288/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772261.096791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772261.096797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772261.160762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772261.288788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772261.480785:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772261.480794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772261.544798:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772261.608770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772261.608771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772261.736783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772261.736892:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772261.800754:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772261.800810:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772261.800826:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772261.928738:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772261.992769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772262.120807:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772262.120807:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772262.184799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772262.312785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772262.504791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772262.504791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772262.568774:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772262.632801:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772262.632816:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772262.760781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772262.760818:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772262.824785:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772262.824787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772262.824853:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772262.952792:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772263.016765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772263.144798:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772263.144798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772263.208815:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772263.336796:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772263.528793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772263.528793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772263.592754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772263.656748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772263.656777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772263.784821:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772263.784845:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772263.848772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772263.848775:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772263.848820:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772263.848847:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772263.976705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772264.040742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772264.168780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772264.168796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772264.232832:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772264.360715:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772264.552704:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772264.552724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772264.616705:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772264.680766:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772264.680809:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772264.808789:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772264.808797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772264.872786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772264.872837:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772265.000741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772265.064779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772265.192828:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772265.192835:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772265.256825:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772265.256911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772265.384758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772265.576781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772265.576860:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772265.640739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772265.704755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772265.704807:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772265.832785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772265.832797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772265.896745:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772265.896805:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772265.896839:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772265.896903:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772266.024745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772266.088764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772266.152787:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772266.152802:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772266.152810:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772266.152827:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1262 00000100:00080000:3.0:1790772266.152845:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772266.152865:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772266.152935:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534412416/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772266.152981:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772266.152990:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772266.152994:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772266.153000:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1262 00000100:00080000:3.0:1790772266.153007:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772266.153015:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772266.153033:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772266.153061:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534412544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772266.153080:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772266.153085:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772266.153088:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772266.153094:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1262 00000100:00080000:3.0:1790772266.153100:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772266.153106:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772266.153113:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772266.153129:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534412672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772266.153143:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772266.153148:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772266.153151:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772266.153163:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1262 00000100:00080000:3.0:1790772266.153176:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772266.153183:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772266.153191:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772266.153206:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534412800/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772266.153221:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772266.153226:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772266.153229:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772266.153236:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1262 00000100:00080000:3.0:1790772266.153241:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772266.153247:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772266.153255:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772266.153278:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534412928/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772266.153295:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772266.153303:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772266.153307:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772266.153326:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1262 00000100:00080000:3.0:1790772266.153337:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772266.153376:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772266.153396:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772266.153430:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534413056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772266.153472:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772266.153478:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772266.153484:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772266.153499:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1262 00000100:00080000:3.0:1790772266.153514:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772266.153524:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772266.153532:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772266.153557:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534413184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772266.153608:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772266.153614:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772266.153616:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772266.153628:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1262 00000100:00080000:3.0:1790772266.153640:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772266.153690:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772266.153697:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772266.153715:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534413312/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772266.153731:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772266.153739:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772266.153748:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772266.153759:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1262 00000100:00080000:3.0:1790772266.153772:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772266.153785:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772266.153794:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772266.153819:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534413440/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772266.216807:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772266.216883:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772266.280824:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772266.280887:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772266.281065:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772266.408802:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772266.600765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772266.600777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772266.664779:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772266.728811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772266.728815:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772266.856785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772266.856787:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772266.920810:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772266.920815:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772267.048744:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772267.112756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772267.240798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772267.240804:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772267.304809:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772267.432782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772267.624780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772267.624783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772267.688774:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772267.752779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772267.752790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772267.880775:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772267.880783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772267.944747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772267.944760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772267.944830:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772267.944833:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772268.072785:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772268.136772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772268.264788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772268.264794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772268.328761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772268.456762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772268.648768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772268.648800:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772268.776773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772268.776800:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772268.904782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772268.904828:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772268.968872:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772268.968873:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772268.968944:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772268.968973:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772269.096770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772269.160753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772269.288774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772269.288777:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772269.352797:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772269.480763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772269.672781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772269.672811:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772269.800771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772269.800811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772269.928817:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772269.928838:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772269.992863:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772269.992882:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772270.120757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772270.184795:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772270.312837:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772270.312840:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772270.376787:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772270.376975:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772270.504799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772270.696826:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772270.696865:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772270.824775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772270.824823:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772270.952790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772270.952813:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772271.016786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772271.016850:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772271.016910:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772271.016946:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772271.144743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772271.208761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772271.272760:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772271.272775:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772271.272781:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772271.272795:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1267 00000100:00080000:3.0:1790772271.272810:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772271.272831:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772271.272901:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534413568/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772271.272944:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772271.272949:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772271.272952:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772271.272957:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1267 00000100:00080000:3.0:1790772271.272963:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772271.272970:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772271.272984:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772271.273004:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534413696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772271.273026:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772271.273033:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772271.273039:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772271.273052:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1267 00000100:00080000:3.0:1790772271.273057:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772271.273063:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772271.273069:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772271.273084:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534413824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772271.273096:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772271.273101:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772271.273103:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772271.273108:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1267 00000100:00080000:3.0:1790772271.273127:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772271.273138:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772271.273152:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772271.273179:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534413952/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772271.273201:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772271.273207:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772271.273211:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772271.273224:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1267 00000100:00080000:3.0:1790772271.273271:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772271.273283:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772271.273296:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772271.273321:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534414080/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772271.273351:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772271.273356:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772271.273359:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772271.273365:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1267 00000100:00080000:3.0:1790772271.273369:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772271.273381:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772271.273388:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772271.273405:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534414208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772271.273420:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772271.273425:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772271.273431:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772271.273445:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1267 00000100:00080000:3.0:1790772271.273451:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772271.273472:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772271.273481:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772271.273500:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534414336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772271.273539:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772271.273553:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772271.273556:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772271.273563:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1267 00000100:00080000:3.0:1790772271.273569:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772271.273577:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772271.273587:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772271.273606:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534414464/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772271.273631:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772271.273636:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772271.273639:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772271.273645:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1267 00000100:00080000:3.0:1790772271.273696:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772271.273707:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772271.273719:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772271.273739:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534414592/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772271.336771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772271.336812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772271.400874:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772271.528746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772271.720810:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772271.720827:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772271.848742:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772271.848779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772271.976828:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772271.976945:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772272.040803:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772272.040976:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772272.041319:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772272.041427:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772272.041444:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772272.168764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772272.232768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772272.360783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772272.360859:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772272.424796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772272.552759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772272.744783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772272.744784:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772272.872771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772272.872782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772273.000785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772273.000809:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772273.064746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772273.064778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772273.064803:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772273.192706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772273.256739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772273.384751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772273.384766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772273.448775:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772273.576720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772273.768827:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772273.768839:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772273.896763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772273.896771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772274.024797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772274.024817:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772274.088803:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772274.088963:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772274.216820:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772274.280763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772274.408761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772274.408771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772274.472803:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772274.600746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772274.792743:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772274.792767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772274.920773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772274.920776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772275.048791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772275.048792:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772275.112771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772275.112826:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772275.112861:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772275.112926:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772275.240771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772275.304777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772275.432817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772275.432894:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00010000:00010000:0.0:1790772275.496814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772275.496850:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772275.624765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772275.816768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772275.816835:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772275.944766:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772275.944782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772276.072805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772276.072843:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772276.136891:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772276.136916:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772276.136976:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772276.136981:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772276.264759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772276.328763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772276.392799:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772276.392824:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772276.392835:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772276.392856:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1272 00000100:00080000:3.0:1790772276.392875:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772276.392898:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772276.392970:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534414720/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772276.393030:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772276.393038:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772276.393043:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772276.393052:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1272 00000100:00080000:3.0:1790772276.393061:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772276.393071:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772276.393088:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772276.393118:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534414848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772276.393136:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772276.393180:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772276.393184:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772276.393193:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1272 00000100:00080000:3.0:1790772276.393201:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772276.393226:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772276.393236:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772276.393261:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534414976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772276.393279:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772276.393285:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772276.393288:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772276.393294:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1272 00000100:00080000:3.0:1790772276.393300:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772276.393308:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772276.393316:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772276.393341:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534415104/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772276.393364:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772276.393374:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772276.393381:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772276.393403:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1272 00000100:00080000:3.0:1790772276.393415:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772276.393436:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772276.393468:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772276.393497:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534415232/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772276.393517:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772276.393522:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772276.393527:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772276.393547:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1272 00000100:00080000:3.0:1790772276.393555:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772276.393568:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772276.393576:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772276.393603:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534415360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772276.393714:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772276.393723:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772276.393731:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772276.393747:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1272 00000100:00080000:3.0:1790772276.393759:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772276.393772:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772276.393781:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772276.393806:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534415488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772276.393828:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772276.393838:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772276.393847:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772276.393857:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1272 00000100:00080000:3.0:1790772276.393862:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772276.393875:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772276.393890:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772276.393916:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534415616/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772276.393938:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772276.393946:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772276.393952:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772276.393968:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1272 00000100:00080000:3.0:1790772276.393980:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772276.393993:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772276.394008:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772276.394041:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534415744/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772276.456754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772276.456794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772276.520790:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772276.520868:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772276.648725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772276.840745:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772276.840760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772276.968785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772276.968820:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772277.096797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772277.096814:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772277.160814:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772277.160814:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772277.288739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772277.352759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772277.480784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772277.480814:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772277.544886:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772277.672761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772277.673850:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772277.864761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772277.864761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772277.992784:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772277.992787:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772278.120771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772278.120845:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772278.184806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772278.184887:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772278.184896:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772278.312756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772278.376790:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772278.504803:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772278.504813:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772278.568814:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772278.696832:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772278.888731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772278.888735:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772279.016760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772279.016772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772279.144760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772279.144809:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772279.208895:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772279.209017:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772279.209060:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772279.336763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772279.400727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772279.528789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772279.528819:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772279.592775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772279.720794:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772279.912734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772279.912762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772280.040786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772280.040798:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772280.168830:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772280.168832:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772280.232858:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772280.233033:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772280.233098:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772280.360745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772280.424755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772280.552782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772280.552800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772280.616750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772280.744760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772280.936781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772280.936822:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772281.064797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772281.064810:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772281.192779:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772281.192823:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772281.256811:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772281.256817:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772281.384762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772281.448756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772281.576799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772281.576800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772281.640849:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772281.768788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772281.960734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772281.960760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772282.088816:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772282.088820:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772282.216802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772282.216805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772282.280785:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772282.280877:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772282.280877:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772282.408773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772282.472766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772282.600745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772282.600823:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772282.664791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772282.792787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772282.984799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772282.984809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772283.112784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772283.112830:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772283.240921:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772283.241729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772283.304779:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772283.304822:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772283.304856:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772283.432804:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772283.496781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772283.624755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772283.624760:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772283.688816:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772283.816720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772284.008767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772284.008798:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772284.136791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772284.136793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772284.264822:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772284.264824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772284.328747:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772284.328780:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772284.328820:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772284.456801:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772284.520819:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772284.648809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772284.648817:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772284.712842:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772284.840754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772285.032786:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772285.032789:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772285.160797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772285.160825:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772285.288777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772285.288849:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772285.352725:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772285.352738:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772285.352769:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772285.352776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772285.480761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772285.544777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772285.672795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772285.672805:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772285.736824:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772285.864787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772286.056759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772286.056763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772286.184769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772286.184808:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772286.312833:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772286.313734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772286.376813:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772286.376830:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772286.376890:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772286.376895:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772286.504762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772286.568763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772286.632847:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772286.632861:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772286.632870:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772286.632891:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1282 00000100:00080000:0.0:1790772286.632914:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772286.632934:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772286.633050:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534417024/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772286.633102:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772286.633109:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772286.633112:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772286.633119:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1282 00000100:00080000:0.0:1790772286.633128:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772286.633139:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772286.633169:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772286.633208:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534417152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772286.633235:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772286.633243:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772286.633248:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772286.633258:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1282 00000100:00080000:0.0:1790772286.633272:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772286.633284:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772286.633301:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772286.633327:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534417280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772286.633343:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772286.633349:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772286.633352:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772286.633358:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1282 00000100:00080000:0.0:1790772286.633366:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772286.633374:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772286.633383:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772286.633403:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534417408/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772286.633437:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772286.633449:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772286.633454:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772286.633466:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1282 00000100:00080000:0.0:1790772286.633473:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772286.633481:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772286.633509:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772286.633531:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534417536/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772286.633545:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772286.633551:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772286.633554:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772286.633566:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1282 00000100:00080000:0.0:1790772286.633578:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772286.633587:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772286.633596:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772286.633618:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534417664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772286.633638:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772286.633647:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772286.633720:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772286.633736:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1282 00000100:00080000:0.0:1790772286.633751:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772286.633773:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772286.633788:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772286.633823:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534417792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772286.633860:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772286.633869:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772286.633876:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772286.633895:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1282 00000100:00080000:0.0:1790772286.633909:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772286.633928:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772286.633937:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772286.633996:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534417920/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772286.634038:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772286.634048:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772286.634054:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772286.634064:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1282 00000100:00080000:0.0:1790772286.634074:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772286.634092:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772286.634102:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772286.634131:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534418048/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772286.696797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772286.696863:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772286.760746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772286.888762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772287.080795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772287.080809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772287.208765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772287.208770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772287.336958:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772287.337734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772287.400823:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772287.400845:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772287.400911:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772287.528738:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772287.592758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772287.720808:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772287.720890:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772287.912750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772288.104758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772288.232869:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772288.232870:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772288.360769:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772288.360845:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772288.424768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772288.424838:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772288.424851:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772288.424907:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772288.552742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772288.616757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772288.744760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772288.744850:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772288.808845:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772288.936762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772289.128845:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772289.256769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772289.256822:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772289.384756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772289.384881:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772289.448804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772289.448807:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772289.448863:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772289.576754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772289.640764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772289.768812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772289.768823:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772289.832726:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772289.960753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772290.152818:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772290.216933:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772290.280770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772290.280825:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772290.408835:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772290.408918:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772290.472827:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772290.472869:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772290.600790:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772290.664765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772290.792859:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772290.792935:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772290.856892:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772290.984784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772291.176846:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772291.240781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772291.304720:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772291.304836:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772291.432752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772291.432973:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772291.496781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772291.496847:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772291.496924:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772291.624743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772291.688762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772291.753099:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534418176:192.168.204.150@tcp:250:kworker.0 00000800:00000100:3.0:1790772291.753405:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772291.753519:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772291/real 1790772291] req@ffff9c4efbcc2a00 x1877759534418176/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772291.753781:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534418176/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772291.753807:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772291.753834:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772291.753874:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534418176:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772291.753944:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534418304:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772291.754021:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772291.754043:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772291/real 1790772291] req@ffff9c4efbcc1500 x1877759534418304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772291.754061:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534418304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772291.754087:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772291.754092:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772291.754098:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534418304:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772291.754107:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534418432:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772291.754133:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772291.754146:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772291/real 1790772291] req@ffff9c4efbcc2680 x1877759534418432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772291.754162:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534418432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772291.754182:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772291.754187:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772291.754192:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534418432:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772291.754199:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534418560:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772291.754221:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772291.754233:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772291/real 1790772291] req@ffff9c4efbcc1c00 x1877759534418560/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772291.754247:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534418560/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772291.754261:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772291.754266:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772291.754270:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534418560:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772291.754279:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534418688:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772291.754298:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772291.754310:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772291/real 1790772291] req@ffff9c4efbcc2300 x1877759534418688/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772291.754324:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534418688/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772291.754337:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772291.754341:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772291.754347:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534418688:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772291.754355:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534418816:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772291.754377:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772291.754389:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772291/real 1790772291] req@ffff9c4efbcc1880 x1877759534418816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772291.754404:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534418816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772291.754425:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772291.754429:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772291.754433:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534418816:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772291.754441:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534418944:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772291.754462:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772291.754474:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772291/real 1790772291] req@ffff9c4efbcc1f80 x1877759534418944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772291.754489:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534418944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772291.754500:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772291.754504:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772291.754508:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534418944:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772291.754516:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534419072:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772291.754536:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772291.754549:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772291/real 1790772291] req@ffff9c4efbcc0700 x1877759534419072/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772291.754564:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534419072/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772291.754578:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772291.754582:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772291.754586:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534419072:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772291.754620:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534419200:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772291.754643:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772291.754691:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772291/real 1790772291] req@ffff9c4efbcc3100 x1877759534419200/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772291.754708:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534419200/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772291.754721:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772291.754726:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772291.754731:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534419200:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772291.816776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772291.816790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772291.880792:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772291.880940:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772292.008785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772292.200906:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772292.264779:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772292.328888:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772292.328890:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772292.456809:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772292.456866:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772292.520779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772292.520877:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772292.520957:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772292.648749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772292.712756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772292.840808:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772292.840827:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772292.904810:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772292.904952:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772293.032780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772293.224864:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772293.288712:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772293.352769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772293.352811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772293.480781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772293.480802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772293.544787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772293.544879:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772293.544908:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772293.672749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772293.736786:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772293.864835:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772293.864895:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772293.928774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772293.928893:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772293.928952:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772294.056734:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772294.248788:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772294.312766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772294.376785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772294.376786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772294.504830:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772294.504877:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772294.568773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772294.568827:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772294.568870:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772294.568876:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772294.696931:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772294.760781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772294.888822:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772294.888829:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772294.952867:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772294.952925:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772295.080811:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772295.272829:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772295.336811:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772295.400770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772295.400776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772295.528897:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772295.528905:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772295.592818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772295.592824:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772295.592942:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772295.592943:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772295.720844:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772295.784756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772295.912801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772295.912807:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00010000:00010000:0.0:1790772295.976909:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772296.104753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772296.296753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772296.360817:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772296.424815:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772296.424838:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772296.552827:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772296.552836:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772296.616904:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772296.616944:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772296.617812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772296.744858:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772296.808801:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772296.873031:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534419328:192.168.204.150@tcp:250:kworker.0 00000800:00000100:3.0:1790772296.873167:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772296.873240:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772296/real 1790772296] req@ffff9c4efbcc0a80 x1877759534419328/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772296.873291:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534419328/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772296.873310:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772296.873321:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772296.873334:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534419328:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772296.873397:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534419456:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772296.873442:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772296.873461:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772296/real 1790772296] req@ffff9c4efbcc0380 x1877759534419456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772296.873511:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534419456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772296.873548:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772296.873555:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772296.873559:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534419456:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772296.873569:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534419584:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772296.873597:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772296.873613:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772296/real 1790772296] req@ffff9c4efbcc3100 x1877759534419584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772296.873641:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534419584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772296.873681:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772296.873687:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772296.873692:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534419584:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772296.873725:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534419712:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772296.873798:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772296.873839:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772296/real 1790772296] req@ffff9c4efbcc0700 x1877759534419712/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772296.873890:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534419712/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772296.873911:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772296.873919:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772296.873929:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534419712:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772296.873957:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534419840:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772296.873998:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772296.874016:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772296/real 1790772296] req@ffff9c4efbcc1f80 x1877759534419840/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772296.874115:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534419840/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772296.874131:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772296.874135:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772296.874139:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534419840:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772296.874164:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534419968:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772296.874183:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772296.874193:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772296/real 1790772296] req@ffff9c4efbcc1880 x1877759534419968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772296.874205:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534419968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772296.874221:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772296.874223:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772296.874226:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534419968:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772296.874232:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534420096:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772296.874247:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772296.874255:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772296/real 1790772296] req@ffff9c4efbcc2300 x1877759534420096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772296.874266:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534420096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772296.874273:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772296.874276:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772296.874279:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534420096:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772296.874284:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534420224:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772296.874298:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772296.874306:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772296/real 1790772296] req@ffff9c4efbcc1c00 x1877759534420224/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772296.874316:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534420224/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772296.874333:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772296.874336:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772296.874339:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534420224:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772296.874344:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534420352:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772296.874358:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772296.874366:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772296/real 1790772296] req@ffff9c4efbcc2680 x1877759534420352/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772296.874376:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534420352/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772296.874385:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772296.874388:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772296.874391:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534420352:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772296.936714:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772296.936824:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00010000:00010000:0.0:1790772297.000959:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772297.128773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772297.320811:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772297.384813:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772297.448863:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772297.448870:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772297.576816:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772297.576863:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772297.640828:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772297.640860:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772297.640897:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772297.768787:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772297.832809:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772297.960792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772297.960847:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772298.024948:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772298.152822:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772298.345776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772298.408766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772298.472806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772298.472816:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772298.600788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772298.600789:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772298.664768:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772298.664780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772298.792811:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772298.856774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772298.984790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772298.984810:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772299.176783:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772299.304863:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772299.368858:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772299.432761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772299.496805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772299.496812:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772299.624810:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772299.624880:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772299.688791:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772299.688826:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772299.688884:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772299.688906:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772299.816769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772299.880813:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772300.008817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772300.008932:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772300.072974:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772300.200849:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772300.328843:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772300.392870:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772300.456787:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772300.520802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772300.520805:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772300.648852:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772300.648855:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772300.712857:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772300.712876:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772300.840751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772300.904837:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772301.032797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772301.032858:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772301.096777:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772301.224805:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772301.352856:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772301.416748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772301.480717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772301.544806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772301.544841:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772301.672784:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772301.672799:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772301.736795:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772301.736810:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772301.736880:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772301.865764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772301.928777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772301.992857:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772301.992869:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772301.992876:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772301.992895:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1297 00000100:00080000:3.0:1790772301.992914:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772301.992934:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772301.993003:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534420480/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772301.993053:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772301.993059:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772301.993062:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772301.993069:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1297 00000100:00080000:3.0:1790772301.993075:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772301.993085:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772301.993102:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772301.993125:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534420608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772301.993181:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772301.993189:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772301.993196:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772301.993208:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1297 00000100:00080000:3.0:1790772301.993219:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772301.993232:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772301.993245:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772301.993281:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534420736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772301.993308:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772301.993313:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772301.993319:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772301.993329:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1297 00000100:00080000:3.0:1790772301.993343:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772301.993373:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772301.993388:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772301.993416:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534420864/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772301.993463:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772301.993473:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772301.993498:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772301.993514:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1297 00000100:00080000:3.0:1790772301.993524:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772301.993551:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772301.993568:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772301.993603:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534420992/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772301.993629:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772301.993725:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772301.993733:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772301.993742:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1297 00000100:00080000:3.0:1790772301.993757:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772301.993773:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772301.993786:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772301.993814:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534421120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772301.993904:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772301.993912:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772301.993916:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772301.993943:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1297 00000100:00080000:3.0:1790772301.993971:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772301.993988:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772301.993996:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772301.994021:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534421248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772301.994037:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772301.994044:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772301.994047:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772301.994058:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1297 00000100:00080000:3.0:1790772301.994077:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772301.994094:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772301.994102:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772301.994126:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534421376/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772301.994183:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772301.994197:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772301.994210:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772301.994233:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1297 00000100:00080000:3.0:1790772301.994262:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772301.994302:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772301.994318:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772301.994347:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534421504/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772302.056793:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772302.056836:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772302.120892:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772302.248811:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772302.376804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772302.440776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772302.504747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772302.568788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772302.568819:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772302.696778:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772302.696815:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772302.760778:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772302.760823:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772302.760868:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772302.889751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772302.952768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772303.080788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772303.080800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772303.144800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772303.272788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772303.400883:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772303.464768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772303.528757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772303.592776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772303.592790:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772303.720804:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772303.720828:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772303.784747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772303.784821:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772303.784852:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772303.912775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772303.976789:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772304.104783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772304.104823:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772304.168795:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772304.296761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772304.424767:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772304.488794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772304.552756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772304.616752:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772304.616800:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772304.744812:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772304.744822:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772304.808786:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772304.808808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772304.808871:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772304.808886:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772304.936816:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772305.000770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772305.128820:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772305.128824:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772305.192835:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772305.320824:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772305.448787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772305.512789:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772305.576760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772305.640816:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772305.640816:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772305.768852:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772305.768863:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772305.832843:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772305.832843:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772305.832889:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772305.960762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772306.024768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772306.152841:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772306.152850:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772306.216770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772306.216901:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772306.344766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772306.472822:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772306.536810:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772306.600760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772306.664795:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772306.664831:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772306.792800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772306.792849:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772306.856773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772306.856886:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772306.984766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772307.048758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772307.112775:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772307.112786:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772307.112794:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772307.112812:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1303 00000100:00080000:3.0:1790772307.112830:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772307.112850:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772307.112913:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534421632/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772307.112957:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772307.112963:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772307.112966:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772307.112972:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1303 00000100:00080000:3.0:1790772307.112977:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772307.112984:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772307.113000:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772307.113036:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534421760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772307.113068:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772307.113076:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772307.113081:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772307.113092:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1303 00000100:00080000:3.0:1790772307.113101:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772307.113133:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772307.113142:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772307.113163:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534421888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772307.113188:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772307.113194:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772307.113198:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772307.113206:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1303 00000100:00080000:3.0:1790772307.113213:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772307.113221:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772307.113229:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772307.113259:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534422016/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772307.113307:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772307.113314:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772307.113317:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772307.113329:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1303 00000100:00080000:3.0:1790772307.113338:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772307.113352:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772307.113362:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772307.113381:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534422144/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772307.113414:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772307.113422:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772307.113427:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772307.113440:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1303 00000100:00080000:3.0:1790772307.113451:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772307.113468:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772307.113482:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772307.113538:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534422272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772307.113615:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772307.113624:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772307.113628:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772307.113638:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1303 00000100:00080000:3.0:1790772307.113645:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772307.113705:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772307.113720:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772307.113749:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534422400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772307.113779:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772307.113784:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772307.113788:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772307.113799:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1303 00000100:00080000:3.0:1790772307.113812:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772307.113826:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772307.113836:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772307.113863:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534422528/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772307.113894:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772307.113903:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772307.113908:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772307.113920:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1303 00000100:00080000:3.0:1790772307.113925:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772307.113931:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772307.113939:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772307.113959:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534422656/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772307.176779:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772307.176790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772307.240824:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772307.368752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772307.496796:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772307.560758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772307.624755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772307.688785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772307.688792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772307.816787:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772307.816800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772307.880758:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772307.880833:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772307.880837:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772308.008761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772308.072783:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772308.200824:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772308.200824:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772308.264799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772308.392784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772308.520724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772308.584726:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772308.648711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772308.712709:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772308.712742:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772308.840729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772308.840740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772308.904713:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772308.904734:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772308.904760:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772309.032703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772309.096735:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772309.224774:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772309.224812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772309.288784:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772309.416772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772309.544799:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772309.608800:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772309.672768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772309.736790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772309.736800:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772309.864777:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772309.864791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772309.928782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772309.928792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772309.928852:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772309.928872:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772310.056799:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772310.120743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772310.248784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772310.248826:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772310.312771:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772310.312837:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772310.312896:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772310.440810:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772310.568881:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772310.632763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772310.696767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772310.760768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772310.760793:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772310.888772:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772310.888820:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772310.952782:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772310.952819:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772310.952859:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772311.080767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772311.144768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772311.272795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772311.272815:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772311.336807:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772311.336892:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772311.464783:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772311.592785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772311.656785:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772311.784792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772311.784798:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772311.912777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772311.912811:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772311.912858:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772311.976799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772311.976810:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772312.104781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772312.168764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772312.232833:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772312.232846:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772312.232860:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772312.232885:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1308 00000100:00080000:0.0:1790772312.232901:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772312.232919:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772312.232970:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534422784/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772312.233013:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772312.233020:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772312.233023:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772312.233029:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1308 00000100:00080000:0.0:1790772312.233034:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772312.233042:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772312.233054:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772312.233073:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534422912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772312.233105:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772312.233117:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772312.233124:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772312.233138:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1308 00000100:00080000:0.0:1790772312.233147:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772312.233163:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772312.233175:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772312.233210:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534423040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772312.233247:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772312.233254:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772312.233257:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772312.233267:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1308 00000100:00080000:0.0:1790772312.233277:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772312.233285:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772312.233295:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772312.233318:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534423168/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772312.233438:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772312.233444:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772312.233447:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772312.233480:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1308 00000100:00080000:0.0:1790772312.233486:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772312.233501:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772312.233512:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772312.233541:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534423296/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772312.233584:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772312.233597:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772312.233601:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772312.233610:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1308 00000100:00080000:0.0:1790772312.233615:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772312.233625:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772312.233641:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772312.233744:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534423424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772312.233783:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772312.233792:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772312.233806:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772312.233830:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1308 00000100:00080000:0.0:1790772312.233849:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772312.233867:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772312.233879:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772312.233913:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534423552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772312.233973:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772312.233979:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772312.233984:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772312.233994:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1308 00000100:00080000:0.0:1790772312.234004:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772312.234020:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772312.234030:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772312.234063:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534423680/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772312.234169:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772312.234197:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772312.234207:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772312.234220:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1308 00000100:00080000:0.0:1790772312.234229:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772312.234244:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772312.234262:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772312.234290:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534423808/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772312.296844:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772312.296848:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772312.360748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772312.360829:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772312.488831:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772312.616814:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772312.680794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772312.808799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772312.808803:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772312.936750:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772312.936764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772313.000747:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772313.000784:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772313.128746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772313.192766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772313.320793:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772313.320815:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772313.384807:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772313.512813:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772313.640779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772313.704805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772313.832786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772313.832792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772313.960780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772313.960782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772314.024747:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772314.024828:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772314.024891:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772314.152756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772314.216746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772314.344772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772314.344812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772314.408765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772314.536756:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772314.664835:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772314.856755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772314.856768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772314.984766:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772314.984779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772315.048813:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772315.048868:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772315.048919:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772315.176760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772315.240735:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772315.368807:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772315.368814:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772315.432923:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772315.433029:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772315.560785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772315.688769:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772315.752876:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772315.880792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772315.880800:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772316.008751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772316.008800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772316.072792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772316.072793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772316.072862:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772316.200766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772316.264763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772316.392781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772316.392875:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772316.456813:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772316.456926:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772316.584785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772316.776806:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772316.904769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772316.904807:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772317.032793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772317.032796:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772317.096780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772317.096811:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772317.096859:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772317.224755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772317.288745:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772317.353043:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534423936:192.168.204.150@tcp:250:kworker.0 00000800:00000100:3.0:1790772317.353169:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772317.353267:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772317/real 1790772317] req@ffff9c4efbcc1500 x1877759534423936/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772317.353325:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534423936/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772317.353378:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772317.353389:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772317.353403:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534423936:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772317.353448:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534424064:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772317.353495:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772317.353539:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772317/real 1790772317] req@ffff9c4efbcc2a00 x1877759534424064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772317.353588:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534424064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772317.353640:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772317.353702:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772317.353721:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534424064:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772317.353741:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534424192:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772317.353787:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772317.353810:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772317/real 1790772317] req@ffff9c4efbcc2680 x1877759534424192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772317.353831:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534424192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772317.353844:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772317.353850:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772317.353855:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534424192:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772317.353863:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534424320:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772317.353887:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772317.353899:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772317/real 1790772317] req@ffff9c4efbcc1c00 x1877759534424320/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772317.353914:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534424320/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772317.353928:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772317.353932:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772317.353936:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534424320:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772317.353944:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534424448:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772317.353963:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772317.353976:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772317/real 1790772317] req@ffff9c4efbcc2300 x1877759534424448/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772317.353992:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534424448/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772317.354005:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772317.354009:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772317.354013:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534424448:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772317.354041:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534424576:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772317.354063:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772317.354076:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772317/real 1790772317] req@ffff9c4efbcc1880 x1877759534424576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772317.354091:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534424576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772317.354112:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772317.354116:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772317.354120:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534424576:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772317.354127:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534424704:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772317.354146:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772317.354158:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772317/real 1790772317] req@ffff9c4efbcc1f80 x1877759534424704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772317.354174:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534424704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772317.354186:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772317.354191:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772317.354196:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534424704:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772317.354203:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534424832:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772317.354222:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772317.354233:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772317/real 1790772317] req@ffff9c4efbcc0700 x1877759534424832/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772317.354247:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534424832/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772317.354260:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772317.354265:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772317.354269:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534424832:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772317.354275:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534424960:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772317.354294:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772317.354305:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772317/real 1790772317] req@ffff9c4efbcc3100 x1877759534424960/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772317.354319:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534424960/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772317.354332:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772317.354336:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772317.354342:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534424960:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772317.416758:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772317.416798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772317.480808:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772317.480925:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772317.608769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772317.800825:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772317.928824:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772317.928826:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772318.056807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772318.056864:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772318.120774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772318.120928:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772318.248753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772318.312746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772318.440804:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772318.440809:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772318.504909:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772318.632744:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772318.824768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772318.888872:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772318.952788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772318.952802:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772319.080787:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772319.080789:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772319.144785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772319.144869:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772319.144877:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772319.144947:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772319.272755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772319.336752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772319.464800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772319.464807:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772319.528841:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772319.528943:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772319.656743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772319.848778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772319.912736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772319.976753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772319.976783:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772320.104788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772320.104794:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772320.168814:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772320.168841:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772320.296754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772320.360776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772320.488854:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772320.488855:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772320.552808:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772320.552925:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772320.680761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772320.872789:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772320.936746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772321.000786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772321.000821:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772321.128780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772321.128787:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772321.192781:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772321.192784:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772321.192840:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772321.320757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772321.384796:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772321.512801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772321.512804:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772321.576816:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772321.576898:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772321.704760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772321.896780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772321.960744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772322.024800:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772322.024802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772322.152877:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772322.152880:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772322.216756:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772322.216850:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772322.216879:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772322.344754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772322.408773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772322.472891:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772322.473046:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534425088:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772322.473246:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534425216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772322.473357:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534425344:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772322.473428:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534425472:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772322.473498:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534425600:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772322.473645:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534425728:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772322.473759:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534425856:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772322.473831:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534425984:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772322.474097:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534426112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772322.536772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772322.536773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772322.600790:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772322.600873:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772322.728781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772322.920769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772322.984765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772323.048767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772323.048776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772323.176753:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772323.176781:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772323.240825:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772323.240840:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772323.240877:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772323.368754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772323.432755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772323.496760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772323.560814:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772323.560892:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772323.624890:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772323.752721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772323.944791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772324.008750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772324.072785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772324.072785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772324.200809:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772324.200816:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772324.264773:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772324.264824:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772324.264876:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772324.264930:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772324.392752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772324.456780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772324.520773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772324.584903:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772324.584907:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772324.776767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772324.968824:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772325.032812:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772325.096782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772325.096789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772325.224851:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772325.224868:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772325.288811:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772325.288851:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772325.288923:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772325.416773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772325.480747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772325.544759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:0.0:1790772325.545078:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:0.0:1790772325.545143:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1790772325.545171:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772325.545282:0:2385:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:0.0:1790772325.545304:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772325.545324:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772325.545349:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:3.0:1790772325.545362:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772322/real 1790772325] req@ffff9c4efbcc2680 x1877759534426112/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1790772325.545373:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772325.545405:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:3.0:1790772325.545425:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534426112/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1790772325.545435:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772325.545456:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:3.0:1790772325.545490:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772325.545504:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772325.545514:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534426112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772325.545532:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772322/real 1790772325] req@ffff9c4efbcc1c00 x1877759534425984/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772325.545563:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534425984/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772325.545585:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772325.545593:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772325.545601:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534425984:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772325.545615:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772322/real 1790772325] req@ffff9c4efbcc2300 x1877759534425856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772325.545643:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534425856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772325.545700:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772325.545708:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772325.545716:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534425856:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772325.545731:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772322/real 1790772325] req@ffff9c4efbcc1880 x1877759534425728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772325.545758:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534425728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772325.545792:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772325.545800:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772325.545806:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534425728:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772325.545820:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772322/real 1790772325] req@ffff9c4efbcc1f80 x1877759534425600/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772325.545845:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534425600/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772325.545862:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772325.545868:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772325.545875:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534425600:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772325.545888:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772322/real 1790772325] req@ffff9c4efbcc0700 x1877759534425472/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772325.545912:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534425472/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772325.545931:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772325.545939:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772325.545947:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534425472:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772325.545962:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772322/real 1790772325] req@ffff9c4efbcc0a80 x1877759534425216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772325.545987:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534425216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772325.546016:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772325.546024:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772325.546031:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534425216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772325.546046:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772322/real 1790772325] req@ffff9c4efbcc3100 x1877759534425344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772325.546072:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534425344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772325.546091:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772325.546098:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772325.546106:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534425344:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772325.546120:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772322/real 1790772325] req@ffff9c4efbcc0380 x1877759534425088/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772325.546516:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534425088/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772325.546540:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772325.546550:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772325.546558:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534425088:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772325.608817:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772325.608821:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772325.672846:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772325.672936:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772325.800792:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772325.992804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772326.056796:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772326.120809:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772326.120809:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772326.248809:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772326.248810:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772326.312794:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772326.312819:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772326.312879:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772326.440770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772326.504756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772326.568751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772326.632800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772326.632835:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772326.696823:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772326.824784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772327.016803:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772327.080781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772327.144774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772327.144792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772327.272785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772327.272815:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772327.336784:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772327.336815:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772327.336868:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772327.464781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772327.528778:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772327.592780:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772327.592791:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772327.592799:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772327.592818:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1323 00000100:00080000:3.0:1790772327.592836:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772327.592856:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772327.592918:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534426240/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772327.592963:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772327.592969:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772327.592972:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772327.592978:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1323 00000100:00080000:3.0:1790772327.592983:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772327.592991:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772327.593007:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772327.593031:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534426368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772327.593055:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772327.593059:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772327.593062:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772327.593069:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1323 00000100:00080000:3.0:1790772327.593076:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772327.593083:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772327.593090:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772327.593106:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534426496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772327.593120:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772327.593125:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772327.593128:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772327.593134:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1323 00000100:00080000:3.0:1790772327.593140:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772327.593145:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772327.593182:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772327.593204:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534426624/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772327.593217:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772327.593224:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772327.593228:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772327.593238:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1323 00000100:00080000:3.0:1790772327.593246:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772327.593258:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772327.593271:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772327.593301:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534426752/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772327.593357:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772327.593375:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772327.593383:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772327.593402:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1323 00000100:00080000:3.0:1790772327.593413:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772327.593432:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772327.593450:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772327.593488:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534426880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772327.593511:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772327.593525:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772327.593535:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772327.593547:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1323 00000100:00080000:3.0:1790772327.593558:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772327.593588:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772327.593608:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772327.593639:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534427008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772327.593760:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772327.593774:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772327.593780:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772327.593801:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1323 00000100:00080000:3.0:1790772327.593815:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772327.593832:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772327.593854:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772327.593893:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534427136/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772327.593923:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772327.594581:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772327.594593:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772327.594628:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1323 00000100:00080000:3.0:1790772327.594646:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772327.594691:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772327.594705:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772327.594742:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534427264/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772327.656785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772327.656818:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772327.720786:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772327.848768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772328.040811:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772328.104791:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772328.168788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772328.168814:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772328.296887:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772328.296888:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772328.360801:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772328.360840:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772328.360913:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772328.488720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772328.552755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772328.680793:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772328.680812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772328.744838:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772328.872758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772329.064800:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772329.128748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772329.192775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772329.192785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772329.320794:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772329.320802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772329.384781:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772329.384873:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772329.384903:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772329.512759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772329.576765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772329.704844:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772329.704874:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772329.896779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772330.088782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772330.152777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772330.216828:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772330.216831:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772330.344790:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772330.344802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772330.408781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772330.408823:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772330.408868:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772330.536753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772330.600806:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772330.728788:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772330.728821:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772330.792830:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772330.792928:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772330.920785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772331.112819:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772331.176785:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772331.240788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772331.240812:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772331.368809:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772331.368809:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772331.432803:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772331.432803:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772331.560777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772331.624794:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772331.752798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772331.752821:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772331.816832:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772331.944750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772332.136797:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772332.200745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772332.264746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772332.264775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772332.392799:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772332.392810:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772332.456821:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772332.456836:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772332.456900:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772332.584786:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772332.648743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772332.712892:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772332.712903:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772332.712911:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772332.712926:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1328 00000100:00080000:0.0:1790772332.712945:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772332.712971:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772332.713050:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534427392/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772332.713132:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772332.713138:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772332.713141:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772332.713148:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1328 00000100:00080000:0.0:1790772332.713154:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772332.713165:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772332.713182:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772332.713210:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534427520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772332.713228:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772332.713233:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772332.713236:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772332.713243:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1328 00000100:00080000:0.0:1790772332.713249:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772332.713257:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772332.713265:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772332.713280:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534427648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772332.713293:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772332.713299:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772332.713301:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772332.713311:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1328 00000100:00080000:0.0:1790772332.713321:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772332.713330:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772332.713337:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772332.713355:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534427776/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772332.713368:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772332.713373:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772332.713376:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772332.713382:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1328 00000100:00080000:0.0:1790772332.713388:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772332.713396:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772332.713407:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772332.713430:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534427904/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772332.713452:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772332.713458:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772332.713461:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772332.713472:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1328 00000100:00080000:0.0:1790772332.713482:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772332.713496:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772332.713506:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772332.713524:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534428032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772332.713558:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772332.713563:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772332.713567:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772332.713574:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1328 00000100:00080000:0.0:1790772332.713578:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772332.713584:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772332.713591:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772332.713611:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534428160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772332.713628:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772332.713634:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772332.713637:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772332.713703:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1328 00000100:00080000:0.0:1790772332.713713:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772332.713741:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772332.713751:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772332.713778:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534428288/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772332.713797:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772332.713812:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772332.713819:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772332.713832:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1328 00000100:00080000:0.0:1790772332.713840:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772332.713848:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772332.713855:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772332.713878:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534428416/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772332.776916:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772332.776916:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772332.840817:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772332.968810:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772333.160811:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772333.224749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772333.288786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772333.288793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772333.416725:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772333.416728:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772333.480726:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772333.480745:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772333.480800:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772333.608703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772333.672695:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772333.800743:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772333.800746:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772333.864708:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772333.992732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772334.184755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772334.248761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772334.312765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772334.312769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772334.440759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772334.440779:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772334.504787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772334.504814:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772334.632735:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772334.696711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772334.824784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772334.824801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772334.888811:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772335.016772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772335.208846:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772335.272728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772335.336774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772335.336830:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772335.464796:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772335.464826:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772335.528847:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772335.528857:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772335.656767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772335.720790:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772335.848781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772335.848868:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772335.912915:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772336.040772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772336.232809:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772336.296749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772336.360765:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772336.360793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772336.488831:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772336.488910:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772336.552802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772336.552804:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772336.552901:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772336.680751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772336.744761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772336.872784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772336.872798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772336.936876:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772337.064749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772337.256758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772337.384813:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772337.384814:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772337.512801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772337.512887:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772337.576767:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772337.576836:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772337.576844:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772337.704755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772337.768777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772337.832872:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772337.832882:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772337.832889:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772337.832906:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1333 00000100:00080000:3.0:1790772337.832924:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772337.832943:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772337.833048:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534428544/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772337.833089:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772337.833095:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772337.833098:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772337.833104:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1333 00000100:00080000:3.0:1790772337.833109:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772337.833116:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772337.833132:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772337.833153:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534428672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772337.833170:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772337.833176:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772337.833184:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772337.833198:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1333 00000100:00080000:3.0:1790772337.833203:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772337.833208:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772337.833215:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772337.833234:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534428800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772337.833250:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772337.833255:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772337.833258:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772337.833265:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1333 00000100:00080000:3.0:1790772337.833283:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772337.833290:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772337.833297:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772337.833318:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534428928/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772337.833332:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772337.833338:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772337.833342:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772337.833349:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1333 00000100:00080000:3.0:1790772337.833354:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772337.833360:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772337.833366:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772337.833384:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534429056/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772337.833407:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772337.833423:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772337.833426:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772337.833436:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1333 00000100:00080000:3.0:1790772337.833443:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772337.833460:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772337.833470:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772337.833497:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534429184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772337.833524:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772337.833530:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772337.833534:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772337.833542:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1333 00000100:00080000:3.0:1790772337.833547:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772337.833560:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772337.833571:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772337.833592:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534429312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772337.833616:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772337.833624:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772337.833627:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772337.833641:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1333 00000100:00080000:3.0:1790772337.833689:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772337.833699:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772337.833713:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772337.833745:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534429440/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772337.833806:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772337.833812:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772337.833816:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772337.833830:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1333 00000100:00080000:3.0:1790772337.833843:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772337.833856:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772337.833868:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772337.833888:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534429568/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772337.896783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772337.896785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772337.960798:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772338.088757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772338.280780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772338.408764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772338.408802:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772338.536776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772338.536792:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772338.600786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772338.600808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772338.600863:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772338.728771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772338.792740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772338.920746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772338.920792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772339.112788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772339.304752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772339.432783:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772339.432791:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772339.560759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772339.560789:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772339.624760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772339.624784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772339.624841:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772339.624850:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772339.752798:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772339.816775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772339.944772:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772339.944792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772340.008847:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772340.136766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772340.328775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772340.392861:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772340.456762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772340.456798:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772340.584776:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772340.584786:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772340.648770:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772340.648861:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772340.648894:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772340.776761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772340.840766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772340.968761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772340.968790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772341.032803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772341.160781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772341.352763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772341.416737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772341.480788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772341.480797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772341.608776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772341.608828:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772341.672780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772341.672784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772341.672873:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772341.800767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772341.864764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772341.992841:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772341.992870:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772342.056810:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772342.184767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772342.376746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772342.440780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772342.504784:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772342.504792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772342.632808:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772342.632809:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772342.696849:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772342.696915:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772342.824763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772342.888761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772342.952790:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772342.952803:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772342.952811:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772342.952826:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1338 00000100:00080000:3.0:1790772342.952848:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772342.952896:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772342.952976:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534429696/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772342.953028:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772342.953035:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772342.953038:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772342.953045:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1338 00000100:00080000:3.0:1790772342.953052:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772342.953060:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772342.953074:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772342.953098:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534429824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772342.953117:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772342.953125:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772342.953132:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772342.953145:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1338 00000100:00080000:3.0:1790772342.953150:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772342.953155:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772342.953164:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772342.953183:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534429952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772342.953197:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772342.953202:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772342.953205:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772342.953211:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1338 00000100:00080000:3.0:1790772342.953217:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772342.953230:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772342.953244:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772342.953262:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534430080/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772342.953276:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772342.953281:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772342.953285:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772342.953292:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1338 00000100:00080000:3.0:1790772342.953299:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772342.953310:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772342.953322:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772342.953340:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534430208/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772342.953356:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772342.953363:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772342.953371:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772342.953385:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1338 00000100:00080000:3.0:1790772342.953410:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772342.953424:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772342.953433:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772342.953451:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534430336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772342.953466:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772342.953470:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772342.953474:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772342.953481:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1338 00000100:00080000:3.0:1790772342.953487:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772342.953498:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772342.953512:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772342.953528:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534430464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772342.953544:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772342.953548:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772342.953552:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772342.953563:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1338 00000100:00080000:3.0:1790772342.953573:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772342.953582:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772342.953591:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772342.953610:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534430592/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772342.953637:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772342.953645:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772342.953683:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772342.953690:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1338 00000100:00080000:3.0:1790772342.953696:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772342.953702:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772342.953712:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772342.953739:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534430720/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772343.016814:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772343.016892:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772343.080849:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772343.208755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772343.400840:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772343.464756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772343.528807:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772343.528876:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772343.656822:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772343.656832:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772343.720875:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772343.720875:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772343.720974:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772343.848752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772343.912782:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772344.040792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772344.040802:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772344.104813:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772344.232822:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772344.424760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772344.488770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772344.552785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772344.552874:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772344.680817:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772344.680817:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772344.744772:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772344.744860:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772344.744886:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772344.872747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772344.936759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772345.064839:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772345.064840:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772345.128779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772345.256761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772345.448746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772345.512782:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772345.576786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772345.576792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772345.704775:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772345.704885:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772345.768778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772345.768846:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772345.768890:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772345.768912:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772345.896765:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772345.960771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772346.088819:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772346.088824:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772346.152761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772346.152790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772346.152945:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772346.280775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772346.344775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772346.408750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772346.472745:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772346.472818:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772346.536809:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772346.600777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772346.600778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772346.728846:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772346.728854:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772346.728917:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772346.792820:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772346.792851:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772346.792852:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772346.792913:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772346.792928:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772346.920803:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772346.984771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772347.048819:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772347.112833:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772347.112834:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772347.112876:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772347.176835:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772347.176866:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772347.176942:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772347.304810:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772347.368820:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772347.432754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772347.496796:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772347.496830:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772347.560812:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772347.624794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772347.624806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772347.752802:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772347.752838:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772347.752842:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772347.816805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772347.816805:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772347.816821:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772347.816865:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772347.944792:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772348.008805:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772348.072775:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772348.073101:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534430848:192.168.204.150@tcp:250:kworker.0 00000800:00000100:2.0:1790772348.073223:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772348.073299:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772348/real 1790772348] req@ffff9c4efac4fb80 x1877759534430848/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772348.073338:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534430848/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772348.073356:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772348.073367:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772348.073379:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534430848:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772348.073445:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534430976:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772348.073496:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772348.073524:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772348/real 1790772348] req@ffff9c4efac4e680 x1877759534430976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772348.073566:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534430976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772348.073629:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772348.073647:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772348.073696:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534430976:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772348.073720:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534431104:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772348.073815:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772348.073853:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772348/real 1790772348] req@ffff9c4efac4ea00 x1877759534431104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772348.073906:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534431104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772348.073966:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772348.073983:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772348.073997:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534431104:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772348.074017:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534431232:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772348.074070:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772348.074092:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772348/real 1790772348] req@ffff9c4efac4ed80 x1877759534431232/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772348.074118:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534431232/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772348.074133:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772348.074138:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772348.074143:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534431232:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772348.074170:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534431360:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772348.074192:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772348.074205:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772348/real 1790772348] req@ffff9c4efac4c700 x1877759534431360/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772348.074221:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534431360/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772348.074234:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772348.074239:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772348.074244:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534431360:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772348.074251:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534431488:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772348.074269:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772348.074281:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772348/real 1790772348] req@ffff9c4efac4e300 x1877759534431488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772348.074296:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534431488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772348.074325:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772348.074330:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772348.074335:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534431488:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772348.074343:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534431616:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772348.074361:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772348.074373:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772348/real 1790772348] req@ffff9c4efac4f480 x1877759534431616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772348.074387:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534431616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772348.074398:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772348.074402:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772348.074406:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534431616:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772348.074413:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534431744:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772348.074431:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772348.074443:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772348/real 1790772348] req@ffff9c4efac4f800 x1877759534431744/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772348.074458:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534431744/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772348.074472:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772348.074476:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772348.074481:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534431744:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772348.074489:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534431872:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772348.074508:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772348.074520:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772348/real 1790772348] req@ffff9c4efac4dc00 x1877759534431872/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772348.074534:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534431872/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772348.074547:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772348.074551:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772348.074556:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534431872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772348.136787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772348.136797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772348.136837:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772348.200846:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772348.200867:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772348.328807:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772348.392798:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772348.456813:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772348.520778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772348.520815:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772348.584784:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772348.648777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772348.648793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772348.776806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772348.776823:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772348.777206:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772348.840811:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772348.840822:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772348.840878:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772348.840913:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772348.840926:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772348.968774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772349.032790:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772349.096786:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772349.160819:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772349.160820:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772349.160885:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772349.224845:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772349.224896:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772349.352826:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772349.416779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772349.480822:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772349.544783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772349.544820:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772349.608766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772349.672792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772349.672816:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772349.800853:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772349.800853:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772349.800950:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772349.864803:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772349.864803:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772349.864857:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772349.864910:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772349.864911:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772349.992759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772350.056791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772350.120796:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772350.184800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772350.184830:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772350.184840:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772350.248803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772350.248842:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772350.376824:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772350.440786:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772350.504815:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772350.568796:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772350.568822:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772350.632800:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772350.696785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772350.824943:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772350.825051:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772350.825779:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772350.888802:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772350.888840:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772350.888849:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772350.888894:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772351.016795:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772351.080794:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772351.144769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772351.208784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772351.208833:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772351.208834:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772351.272815:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772351.272889:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772351.400780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772351.464792:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772351.528813:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772351.592810:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772351.592820:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772351.656806:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772351.720851:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772351.720852:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772351.848869:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772351.848995:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772351.849114:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772351.912814:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772351.912826:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772351.912839:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772351.912892:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772351.912900:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772352.040787:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772352.104798:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772352.168866:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772352.232794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772352.232799:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772352.232799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772352.296845:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772352.296849:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772352.424756:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772352.488759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772352.552764:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772352.616771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772352.616801:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772352.680842:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772352.744751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772352.744804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772352.872884:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772352.872935:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772352.873023:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772352.936785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772352.936817:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772352.936817:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772352.936874:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772352.936885:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772353.064772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772353.128757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772353.192795:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772353.193113:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534432000:192.168.204.150@tcp:250:kworker.0 00000800:00000100:2.0:1790772353.193270:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772353.193364:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772353/real 1790772353] req@ffff9c4efac4df80 x1877759534432000/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772353.193484:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534432000/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772353.193530:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772353.193551:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772353.193560:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534432000:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772353.193611:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534432128:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772353.193759:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772353.193792:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772353/real 1790772353] req@ffff9c4efac4dc00 x1877759534432128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772353.193836:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534432128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772353.193894:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772353.193921:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772353.193941:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534432128:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772353.193979:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534432256:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772353.194074:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772353.194131:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772353/real 1790772353] req@ffff9c4efac4f800 x1877759534432256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772353.194170:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534432256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772353.194193:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772353.194200:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772353.194205:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534432256:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772353.194215:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534432384:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772353.194252:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772353.194268:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772353/real 1790772353] req@ffff9c4efac4f480 x1877759534432384/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772353.194283:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534432384/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772353.194298:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772353.194302:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772353.194307:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534432384:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772353.194315:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534432512:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772353.194337:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772353.194349:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772353/real 1790772353] req@ffff9c4efac4e300 x1877759534432512/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772353.194364:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534432512/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772353.194376:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772353.194381:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772353.194386:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534432512:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772353.194417:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534432640:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772353.194439:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772353.194452:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772353/real 1790772353] req@ffff9c4efac4c700 x1877759534432640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772353.194466:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534432640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772353.194487:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772353.194491:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772353.194496:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534432640:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772353.194504:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534432768:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772353.194524:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772353.194536:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772353/real 1790772353] req@ffff9c4efac4ed80 x1877759534432768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772353.194551:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534432768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772353.194562:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772353.194566:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772353.194570:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534432768:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772353.194578:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534432896:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772353.194597:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772353.194611:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772353/real 1790772353] req@ffff9c4efac4ea00 x1877759534432896/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772353.194626:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534432896/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772353.194640:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772353.194645:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772353.194706:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534432896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772353.194715:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534433024:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772353.194742:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772353.194759:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772353/real 1790772353] req@ffff9c4efac4e680 x1877759534433024/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772353.194774:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534433024/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772353.194788:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772353.194793:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772353.194798:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534433024:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772353.256780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772353.256795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772353.256801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772353.320793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772353.320885:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772353.448748:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772353.512783:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772353.576805:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772353.640752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772353.640781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772353.704792:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772353.768757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772353.768771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772353.896926:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772353.896927:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772353.897137:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772353.960779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772353.960786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772353.960835:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772353.960866:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772353.960897:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772354.088769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772354.152773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772354.216737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772354.280767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772354.280788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772354.280816:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772354.344757:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772354.344823:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772354.472768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772354.472842:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772354.536771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772354.600763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772354.664799:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772354.664802:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772354.792769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772354.792769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772354.920845:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772354.920861:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772354.920865:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772354.984777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772354.984784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772354.984825:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772354.984880:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772354.984896:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772355.112772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772355.176754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772355.240762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772355.304779:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772355.304803:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772355.304804:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772355.368810:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772355.368839:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772355.496739:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772355.496792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772355.560743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772355.624742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772355.688763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772355.688808:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772355.816753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772355.816767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772355.944747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772355.944782:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772355.944799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772356.008781:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772356.008791:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772356.008832:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772356.008862:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772356.008890:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772356.136767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772356.200777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772356.264741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772356.328776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772356.328804:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772356.328807:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772356.392786:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772356.392821:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772356.392904:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772356.520773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772356.520778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772356.584752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772356.648750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772356.712771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772356.712784:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772356.712864:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772356.840766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772356.840787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772356.968785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772356.968786:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772356.968804:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772357.032789:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772357.032790:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772357.032817:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772357.032846:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772357.032863:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772357.160771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772357.224757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772357.288733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772357.352753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772357.352790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772357.352804:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772357.416794:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772357.416842:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772357.544736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772357.544771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772357.608760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772357.672739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772357.736763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772357.736763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772357.736824:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772357.864790:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772357.864863:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772357.992787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772357.992806:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772357.992808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772358.056782:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772358.056808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772358.056810:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772358.056872:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772358.056899:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772358.184761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772358.248741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772358.312771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772358.312841:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772358.312851:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772358.312858:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772358.312874:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1354 00000100:00080000:3.0:1790772358.312891:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772358.312910:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772358.312975:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534433152/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772358.313030:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772358.313039:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772358.313046:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772358.313057:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1354 00000100:00080000:3.0:1790772358.313066:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772358.313081:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772358.313090:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534433152:192.168.204.150@tcp:250:kworker.0 00000008:00100000:3.0:1790772358.313105:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772358.313135:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534433280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772358.313153:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772358.313159:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772358.313162:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772358.313169:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1354 00000100:00080000:3.0:1790772358.313189:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772358.313197:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772358.313206:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790772358.313209:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772358.313230:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534433408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772358.313245:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772358.313250:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772358.313253:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772358.313259:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1354 00000100:00080000:3.0:1790772358.313264:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772358.313271:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772358.313279:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772358.313283:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772358/real 1790772358] req@ffff9c4ef73c2d80 x1877759534433152/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772358.313302:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534433536/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772358.313317:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772358.313320:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534433152/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772358.313326:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772358.313333:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772358.313348:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1354 00000100:00080000:3.0:1790772358.313355:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772358.313358:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772358.313365:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772358.313373:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:3.0:1790772358.313376:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772358.313384:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534433152:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772358.313398:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534433664/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772358.313411:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772358.313416:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772358.313418:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772358.313425:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1354 00000100:00080000:3.0:1790772358.313430:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772358.313435:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534433280:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772358.313442:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772358.313452:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772358.313472:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534433792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772358.313487:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772358.313489:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772358.313496:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772358.313501:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772358.313511:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1354 00000100:00100000:2.0:1790772358.313515:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772358/real 1790772358] req@ffff9c4ef73c0e00 x1877759534433280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772358.313519:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772358.313535:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772358.313551:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772358.313586:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534433280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772358.313612:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534433920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772358.313619:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772358.313635:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772358.313636:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772358.313646:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1790772358.313682:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534433280:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772358.313694:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534433408:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772358.313703:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772358.313712:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1354 00000100:00080000:3.0:1790772358.313723:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772358.313730:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772358.313737:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772358.313750:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772358.313759:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772358/real 1790772358] req@ffff9c4ef73c0000 x1877759534433408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772358.313775:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534434048/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772358.313787:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534433408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772358.313825:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772358.313826:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772358.313839:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772358.313842:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1790772358.313849:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534433408:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772358.313853:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772358.313864:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534433536:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772358.313870:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1354 00000100:00080000:3.0:1790772358.313883:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772358.313902:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772358.313906:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790772358.313915:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772358.313928:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772358/real 1790772358] req@ffff9c4ef73c1c00 x1877759534433536/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772358.313944:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534434176/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772358.313951:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534433536/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772358.313996:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772358.314006:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772358.314017:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534433536:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772358.314032:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534433664:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772358.314077:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772358.314102:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772358/real 1790772358] req@ffff9c4ef73c2a00 x1877759534433664/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772358.314119:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534433664/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772358.314133:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772358.314138:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772358.314143:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534433664:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772358.314173:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534433792:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772358.314201:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772358.314215:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772358/real 1790772358] req@ffff9c4ef73c0700 x1877759534433792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772358.314230:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534433792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772358.314252:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772358.314257:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772358.314262:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534433792:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772358.314269:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534433920:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772358.314289:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772358.314301:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772358/real 1790772358] req@ffff9c4ef73c2300 x1877759534433920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772358.314316:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534433920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772358.314328:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772358.314333:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772358.314338:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534433920:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772358.314345:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534434048:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772358.314363:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772358.314375:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772358/real 1790772358] req@ffff9c4ef73c0a80 x1877759534434048/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772358.314390:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534434048/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772358.314405:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772358.314409:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772358.314413:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534434048:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772358.314421:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534434176:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772358.314441:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772358.314453:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772358/real 1790772358] req@ffff9c4ef73c1500 x1877759534434176/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772358.314468:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534434176/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772358.314480:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772358.314484:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772358.314490:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534434176:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772358.376795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772358.376814:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772358.376903:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772358.440806:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772358.440859:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772358.440918:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772358.568775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772358.568788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772358.632760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772358.696787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772358.760761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772358.760806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772358.760819:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772358.888771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772358.888792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772359.016801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772359.016820:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772359.016846:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772359.080785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772359.080862:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772359.080862:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772359.080946:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772359.208762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772359.272758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772359.336769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772359.400771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772359.400771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772359.400791:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772359.464792:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772359.464847:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772359.464890:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772359.592791:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772359.592803:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772359.656764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772359.720835:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772359.784762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772359.784774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772359.784801:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772359.912774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772359.912785:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772360.040769:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772360.040790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772360.040809:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772360.104768:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772360.104801:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772360.104855:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772360.104967:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772360.104995:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772360.232772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772360.296759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772360.360749:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772360.424763:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772360.424777:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772360.424786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772360.488778:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772360.488814:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772360.488932:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772360.616766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772360.616777:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772360.680752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772360.744883:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772360.808780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772360.808781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772360.808793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772360.936774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772360.936805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772361.064772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772361.064799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772361.064822:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772361.128764:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772361.128791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772361.128805:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772361.128840:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772361.128848:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772361.256753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772361.320745:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772361.384762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772361.448773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772361.448775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772361.448787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772361.512782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772361.512836:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772361.512954:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772361.640760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772361.640764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772361.704758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772361.768741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772361.832755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772361.832776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772361.832807:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772361.960761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772361.960793:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772362.088814:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772362.088815:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772362.088816:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772362.152821:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772362.152821:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772362.152825:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772362.152886:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772362.152895:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772362.280755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772362.344752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772362.408761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772362.472782:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772362.472878:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772362.472885:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772362.536809:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772362.536854:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772362.537047:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772362.664775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772362.664781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772362.728789:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772362.792792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772362.856747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772362.856765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772362.856788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772362.984791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772362.984801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772363.112748:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772363.112801:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772363.112802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772363.176826:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772363.176836:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772363.176837:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772363.176889:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772363.176893:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772363.304776:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772363.368773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772363.432760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772363.432787:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772363.432801:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772363.432811:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772363.432832:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1359 00000100:00080000:3.0:1790772363.432853:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772363.432878:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772363.432948:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534434304/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772363.432994:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772363.432999:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772363.433003:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772363.433009:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1359 00000100:00080000:3.0:1790772363.433016:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772363.433026:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772363.433056:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534434304:192.168.204.150@tcp:250:kworker.0 00000008:00100000:3.0:1790772363.433057:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772363.433088:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534434432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772363.433105:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772363.433110:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772363.433114:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772363.433120:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1359 00000100:00080000:3.0:1790772363.433126:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772363.433155:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772363.433164:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790772363.433179:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772363.433187:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534434560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772363.433202:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772363.433208:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772363.433212:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772363.433218:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1359 00000100:00080000:3.0:1790772363.433223:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772363.433230:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772363.433238:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772363.433255:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534434688/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772363.433263:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772363/real 1790772363] req@ffff9c4ef73c3800 x1877759534434304/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772363.433283:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772363.433305:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534434304/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772363.433308:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772363.433318:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772363.433340:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772363.433342:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1359 00000100:00080000:2.0:1790772363.433355:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772363.433360:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772363.433365:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534434304:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772363.433380:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772363.433394:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772363.433408:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534434432:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772363.433422:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534434816/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772363.433442:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772363.433448:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772363.433451:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790772363.433488:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772363.433493:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1359 00000100:00080000:3.0:1790772363.433508:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772363.433520:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772363/real 1790772363] req@ffff9c4ef73c1500 x1877759534434432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772363.433527:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772363.433548:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772363.433574:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534434432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772363.433590:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534434944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772363.433599:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772363.433609:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772363.433613:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772363.433621:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1790772363.433628:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534434432:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772363.433630:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772363.433688:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534434560:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772363.433700:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1359 00000100:00080000:3.0:1790772363.433719:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772363.433727:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772363.433742:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772363.433753:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772363/real 1790772363] req@ffff9c4ef73c0a80 x1877759534434560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772363.433760:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772363.433782:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534434560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772363.433797:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534435072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772363.433801:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772363.433817:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772363.433831:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534434560:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772363.433840:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772363.433848:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534434688:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772363.433850:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772363.433859:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772363.433869:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1359 00000100:00080000:3.0:1790772363.433877:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772363.433891:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772363.433898:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790772363.433905:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772363.433921:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772363/real 1790772363] req@ffff9c4ef73c2300 x1877759534434688/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772363.433931:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534435200/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772363.433958:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772363.433960:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534434688/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772363.433970:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772363.433977:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772363.433994:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1359 00000100:00080000:3.0:1790772363.434006:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772363.434019:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772363.434021:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790772363.434027:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:2.0:1790772363.434035:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772363.434045:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534434688:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772363.434052:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534435328/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772363.434089:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534434816:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772363.434118:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772363.434133:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772363/real 1790772363] req@ffff9c4ef73c0700 x1877759534434816/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772363.434149:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534434816/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772363.434162:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772363.434167:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772363.434171:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534434816:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772363.434178:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534434944:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772363.434197:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772363.434209:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772363/real 1790772363] req@ffff9c4ef73c2a00 x1877759534434944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772363.434225:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534434944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772363.434246:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772363.434250:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772363.434256:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534434944:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772363.434263:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534435072:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772363.434283:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772363.434295:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772363/real 1790772363] req@ffff9c4ef73c1c00 x1877759534435072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772363.434310:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534435072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772363.434321:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772363.434325:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772363.434329:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534435072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772363.434336:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534435200:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772363.434356:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772363.434368:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772363/real 1790772363] req@ffff9c4ef73c0000 x1877759534435200/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772363.434383:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534435200/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772363.434397:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772363.434401:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772363.434405:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534435200:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772363.434413:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534435328:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772363.434432:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772363.434445:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772363/real 1790772363] req@ffff9c4ef73c0e00 x1877759534435328/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772363.434483:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534435328/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772363.434497:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772363.434501:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772363.434506:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534435328:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772363.496832:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772363.496834:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772363.496835:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772363.560828:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772363.560866:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772363.560921:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772363.688762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772363.688762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772363.752754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772363.816832:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772363.880771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772363.880779:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772363.880816:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772364.008787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772364.008835:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772364.136818:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772364.136819:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772364.136819:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772364.200777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772364.200795:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772364.200869:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772364.200871:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772364.200917:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772364.328741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772364.392776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772364.456754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772364.520753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772364.520869:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772364.520907:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772364.584780:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772364.584873:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772364.584986:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772364.712828:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772364.712830:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772364.776787:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772364.840801:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772364.904754:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772364.904767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772364.904791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772365.032782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772365.032830:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772365.160784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772365.160808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772365.160815:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772365.224815:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772365.224838:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772365.224858:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772365.224896:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772365.224908:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772365.352739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772365.416775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772365.480766:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772365.544789:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772365.544821:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772365.544865:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772365.608858:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772365.608863:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772365.609018:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772365.736764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772365.736781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772365.800768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772365.864805:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772365.928775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772365.928781:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772365.928783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772366.056825:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772366.056833:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772366.184793:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772366.184803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772366.184803:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772366.248795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772366.248803:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772366.248854:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772366.248878:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772366.248937:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772366.376768:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772366.440750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772366.504751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772366.568773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772366.568821:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772366.568826:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772366.632783:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772366.632823:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772366.632921:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772366.760772:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772366.760780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772366.824763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772366.888820:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772366.952771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772366.952775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772366.952779:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772367.080783:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772367.080798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772367.208805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772367.208811:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772367.208816:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772367.272831:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772367.272833:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772367.272850:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772367.272902:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772367.400832:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772367.464776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772367.528786:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772367.592817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772367.592875:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772367.592880:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772367.656800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772367.656811:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772367.784772:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772367.784775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772367.848762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772367.912890:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772367.976773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772367.976800:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772367.976804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772368.104888:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772368.104890:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772368.232840:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772368.232862:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772368.232869:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772368.296784:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772368.296812:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772368.296853:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772368.296868:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772368.296912:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772368.424763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772368.488754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772368.552766:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772368.552778:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1790772368.552781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772368.552786:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772368.552808:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1364 00000100:00080000:3.0:1790772368.552825:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772368.552845:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772368.552911:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534435456/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772368.552967:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772368.552973:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772368.552976:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772368.552983:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1364 00000100:00080000:3.0:1790772368.552988:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772368.552997:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772368.553015:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772368.553046:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534435456:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772368.553046:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534435584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772368.553076:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772368.553081:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772368.553084:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772368.553091:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1364 00000100:00080000:3.0:1790772368.553098:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772368.553104:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772368.553112:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772368.553131:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534435712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772368.553145:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772368.553150:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772368.553154:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772368.553161:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1364 00000100:00080000:3.0:1790772368.553166:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772368.553172:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772368.553180:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:2.0:1790772368.553184:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772368.553200:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534435840/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772368.553216:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772368.553221:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772368.553225:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772368.553231:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1364 00000100:00080000:3.0:1790772368.553236:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772368.553241:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772368.553248:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772368.553265:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534435968/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772368.553274:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772368/real 1790772368] req@ffff9c4ef73c2d80 x1877759534435456/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772368.553287:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772368.553303:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772368.553310:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772368.553327:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1364 00000100:00080000:3.0:1790772368.553332:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772368.553340:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772368.553350:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772368.553351:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534435456/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772368.553396:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534436096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772368.553410:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772368.553412:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772368.553423:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772368.553426:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772368.553428:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772368.553446:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534435456:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772368.553446:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1364 00000100:00080000:3.0:1790772368.553460:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772368.553470:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772368.553477:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772368.553492:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534436224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772368.553506:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534435584:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772368.553512:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772368.553520:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772368.553523:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772368.553545:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1364 00000800:00000100:2.0:1790772368.553556:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772368.553558:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772368.553578:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772368.553585:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772368/real 1790772368] req@ffff9c4ef73c0e00 x1877759534435584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772368.553595:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772368.553610:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534435584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772368.553634:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534436352/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772368.553638:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772368.553683:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772368.553690:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534435584:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772368.553704:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534435712:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772368.553723:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772368.553729:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772368.553734:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772368.553744:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1364 00000800:00000100:2.0:1790772368.553751:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772368.553757:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772368.553773:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772368.553779:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772368/real 1790772368] req@ffff9c4ef73c0000 x1877759534435712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772368.553787:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772368.553810:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534435712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772368.553823:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534436480/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772368.553829:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772368.553846:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772368.553860:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534435712:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772368.553872:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534435840:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772368.553899:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772368.553914:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772368/real 1790772368] req@ffff9c4ef73c1c00 x1877759534435840/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772368.553930:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534435840/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772368.553944:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772368.553949:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772368.553953:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534435840:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772368.553961:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534435968:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772368.553983:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772368.554004:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772368/real 1790772368] req@ffff9c4ef73c2a00 x1877759534435968/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772368.554028:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534435968/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772368.554048:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772368.554057:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772368.554066:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534435968:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772368.554079:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534436096:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772368.554116:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772368.554133:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772368/real 1790772368] req@ffff9c4ef73c0700 x1877759534436096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772368.554149:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534436096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772368.554172:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772368.554177:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772368.554182:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534436096:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772368.554214:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534436224:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772368.554247:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772368.554269:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772368/real 1790772368] req@ffff9c4ef73c2300 x1877759534436224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772368.554297:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534436224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772368.554317:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772368.554325:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772368.554334:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534436224:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772368.554343:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534436352:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772368.554403:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772368.554421:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772368/real 1790772368] req@ffff9c4ef73c0a80 x1877759534436352/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772368.554438:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534436352/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772368.554453:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772368.554457:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772368.554463:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534436352:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772368.554472:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534436480:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772368.554495:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772368.554507:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772368/real 1790772368] req@ffff9c4ef73c1500 x1877759534436480/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772368.554522:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534436480/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772368.554535:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772368.554539:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772368.554543:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534436480:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772368.616773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772368.616789:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772368.616795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772368.680793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772368.680813:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772368.808708:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772368.808719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772368.872735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772368.936724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772369.000718:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772369.000718:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772369.000718:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772369.128707:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772369.128717:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772369.256704:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772369.256724:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772369.256724:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772369.320771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772369.320795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772369.320795:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772369.320846:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772369.448752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772369.512794:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772369.576709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772369.640812:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772369.640816:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772369.640816:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772369.704787:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772369.704828:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772369.832792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772369.832793:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772369.896745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772369.960795:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772370.024781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772370.024789:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772370.024801:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772370.152752:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772370.152782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772370.280784:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772370.280785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772370.280797:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772370.344776:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772370.344787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772370.344791:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772370.344844:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772370.344867:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772370.472744:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772370.536769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772370.600786:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772370.664766:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772370.664785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772370.664867:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772370.728847:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772370.728877:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772370.856780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772370.856780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772370.920761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772370.984777:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772371.048755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772371.048763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772371.048798:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772371.176767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772371.176772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772371.304758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772371.304797:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772371.304798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772371.368769:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772371.368785:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772371.368799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772371.368863:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772371.496757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772371.560758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772371.624735:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772371.688753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772371.688784:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772371.688796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772371.752740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772371.752890:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772371.880753:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772371.880760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772371.944801:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772372.008762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772372.072804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772372.072809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772372.072809:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772372.200772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772372.200792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772372.328784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772372.328791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772372.328798:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772372.392755:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772372.392848:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772372.392848:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772372.392852:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772372.392907:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772372.520783:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772372.584758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772372.648749:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772372.712777:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772372.712790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772372.712804:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772372.776802:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772372.776806:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772372.776949:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772372.904773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772372.904795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772372.968752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772373.032793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772373.096763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772373.096776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772373.096799:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772373.224787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772373.224806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772373.352762:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772373.352796:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772373.352808:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772373.416778:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772373.416802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772373.416830:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772373.416867:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772373.416893:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772373.544766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772373.608791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772373.672768:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772373.672783:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772373.672791:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772373.672793:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772373.672814:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1369 00000100:00080000:3.0:1790772373.672837:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772373.672863:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772373.672939:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534436608/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772373.672984:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772373.672990:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772373.672993:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772373.672999:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1369 00000100:00080000:3.0:1790772373.673007:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772373.673017:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772373.673035:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772373.673046:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534436608:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772373.673072:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534436736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772373.673100:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772373.673109:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772373.673115:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772373.673125:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1369 00000100:00080000:3.0:1790772373.673138:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772373.673149:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772373.673159:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790772373.673164:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772373.673188:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534436864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772373.673206:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772373.673211:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772373.673215:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772373.673223:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1369 00000100:00080000:3.0:1790772373.673230:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772373.673243:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772373/real 1790772373] req@ffff9c4ef73c3800 x1877759534436608/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772373.673244:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772373.673285:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772373.673308:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534436608/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772373.673315:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534436992/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772373.673355:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772373.673358:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772373.673368:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772373.673370:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772373.673374:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772373.673383:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534436608:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772373.673385:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1369 00000100:00080000:3.0:1790772373.673400:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772373.673409:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772373.673416:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772373.673432:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534437120/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772373.673438:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534436736:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772373.673468:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772373.673475:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772373.673479:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772373.673488:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1369 00000800:00000100:2.0:1790772373.673492:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772373.673498:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772373.673513:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772373.673523:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772373.673524:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772373/real 1790772373] req@ffff9c4ef73c1500 x1877759534436736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772373.673549:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534437248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772373.673553:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534436736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772373.673589:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772373.673590:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772373.673605:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772373.673606:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772373.673613:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772373.673620:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534436736:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772373.673629:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1369 00000100:00080000:3.0:1790772373.673637:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772373.673641:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534436864:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772373.673649:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772373.673697:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772373.673724:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534437376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772373.673740:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772373.673741:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772373.673752:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772373.673757:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772373.673767:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1369 00000100:00100000:2.0:1790772373.673770:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772373/real 1790772373] req@ffff9c4ef73c0a80 x1877759534436864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772373.673774:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772373.673790:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772373.673799:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772373.673802:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534436864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772373.673819:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790772373.673824:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534437504/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772373.673828:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772373.673841:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534436864:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772373.673865:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534436992:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772373.673899:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772373.673907:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772373.673916:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772373.673924:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772373.673930:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772373/real 1790772373] req@ffff9c4ef73c2300 x1877759534436992/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772373.673940:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1369 00000100:00080000:3.0:1790772373.673955:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772373.673966:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534436992/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772373.673972:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772373.673989:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:2.0:1790772373.673995:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772373.674004:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772373.674010:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534436992:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772373.674019:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534437632/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772373.674053:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534437120:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772373.674084:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772373.674097:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772373/real 1790772373] req@ffff9c4ef73c0700 x1877759534437120/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772373.674113:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534437120/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772373.674125:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772373.674129:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772373.674134:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534437120:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772373.674142:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534437248:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772373.674162:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772373.674175:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772373/real 1790772373] req@ffff9c4ef73c2a00 x1877759534437248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772373.674192:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534437248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772373.674212:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772373.674216:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772373.674221:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534437248:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772373.674228:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534437376:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772373.674248:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772373.674290:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772373/real 1790772373] req@ffff9c4ef73c1c00 x1877759534437376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772373.674307:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534437376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772373.674319:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772373.674323:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772373.674328:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534437376:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772373.674336:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534437504:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772373.674359:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772373.674372:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772373/real 1790772373] req@ffff9c4ef73c0000 x1877759534437504/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772373.674387:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534437504/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772373.674411:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772373.674416:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772373.674421:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534437504:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772373.674428:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534437632:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772373.674449:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772373.674460:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772373/real 1790772373] req@ffff9c4ef73c0e00 x1877759534437632/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772373.674475:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534437632/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772373.674488:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772373.674492:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772373.674496:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534437632:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772373.736750:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772373.736804:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772373.736821:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772373.800820:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772373.800867:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772373.800882:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772373.928762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772373.928762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772373.992764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772374.056782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772374.120764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772374.120776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772374.120793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772374.248791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772374.248805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772374.376764:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772374.376809:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772374.376818:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772374.440787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772374.440791:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772374.440825:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772374.440883:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772374.568753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772374.632840:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772374.696757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772374.760776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772374.760803:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772374.760804:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772374.824813:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772374.824840:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772374.824892:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772374.952770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772374.952799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772375.016766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772375.080812:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772375.144757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772375.144805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772375.144805:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772375.272785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772375.272787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772375.400774:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772375.400791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772375.400797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772375.464763:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772375.464795:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772375.464809:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772375.464835:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772375.464863:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772375.592751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772375.656751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772375.720783:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772375.784769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772375.784771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772375.784782:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772375.848786:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772375.848873:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772375.849007:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772375.976764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772375.976786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772376.040810:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772376.104767:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772376.168773:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772376.168780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772376.168786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772376.296761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772376.296772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772376.424783:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772376.424802:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772376.424806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772376.488788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772376.488839:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772376.488843:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772376.488863:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772376.616755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772376.680779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772376.744757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772376.808772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772376.808797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772376.808827:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772376.872770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772376.872836:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772377.000743:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772377.000776:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772377.064739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772377.128792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772377.192755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772377.192780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772377.192788:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772377.320778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772377.320790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772377.448783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772377.448803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772377.448805:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772377.512755:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772377.512782:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772377.512814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772377.512830:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772377.512870:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772377.640756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772377.704775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772377.768780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772377.832834:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772377.832835:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772377.832850:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772377.896775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772377.896868:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772377.896949:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772378.024761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772378.024786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772378.088751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772378.152798:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772378.216765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772378.216768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772378.216795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772378.344798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772378.344808:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772378.472791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772378.472791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772378.472802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772378.536839:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772378.536840:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772378.536875:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772378.536972:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772378.536982:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772378.664830:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772378.728776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772378.792762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772378.793116:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534437760:192.168.204.150@tcp:250:kworker.0 00000800:00000100:2.0:1790772378.793271:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772378.793350:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772378/real 1790772378] req@ffff9c4efac4fb80 x1877759534437760/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772378.793418:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534437760/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772378.793471:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772378.793493:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772378.793513:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534437760:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772378.793565:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534437888:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772378.793631:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772378.793707:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772378/real 1790772378] req@ffff9c4efac4e680 x1877759534437888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772378.793732:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534437888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772378.793787:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772378.793802:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772378.793818:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534437888:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772378.793835:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534438016:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772378.793870:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772378.793891:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772378/real 1790772378] req@ffff9c4efac4ea00 x1877759534438016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772378.793923:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534438016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772378.793973:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772378.793989:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772378.794002:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534438016:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772378.794026:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534438144:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772378.794062:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772378.794082:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772378/real 1790772378] req@ffff9c4efac4ed80 x1877759534438144/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772378.794120:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534438144/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772378.794140:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772378.794151:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772378.794187:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534438144:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772378.794223:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534438272:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772378.794267:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772378.794291:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772378/real 1790772378] req@ffff9c4efac4c700 x1877759534438272/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772378.794327:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534438272/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772378.794351:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772378.794357:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772378.794371:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534438272:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772378.794379:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534438400:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772378.794402:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772378.794414:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772378/real 1790772378] req@ffff9c4efac4e300 x1877759534438400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772378.794429:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534438400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772378.794462:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772378.794466:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772378.794471:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534438400:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772378.794479:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534438528:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772378.794498:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772378.794510:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772378/real 1790772378] req@ffff9c4efac4f480 x1877759534438528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772378.794525:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534438528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772378.794536:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772378.794540:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772378.794546:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534438528:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772378.794554:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534438656:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772378.794574:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772378.794586:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772378/real 1790772378] req@ffff9c4efac4f800 x1877759534438656/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772378.794601:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534438656/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772378.794615:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772378.794619:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772378.794623:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534438656:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772378.794679:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534438784:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772378.794716:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772378.794731:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772378/real 1790772378] req@ffff9c4efac4dc00 x1877759534438784/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772378.794748:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534438784/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772378.794761:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772378.794765:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772378.794770:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534438784:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772378.856784:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772378.856838:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772378.856907:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772378.920775:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772378.920831:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772378.920938:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772379.048771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772379.048777:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772379.112769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772379.176797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772379.240745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772379.240778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772379.240789:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772379.368769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772379.368776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772379.496778:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772379.496785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772379.496865:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772379.560786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772379.560801:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772379.560826:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772379.560872:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772379.688760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772379.752752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772379.816751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772379.880759:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772379.880772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772379.880796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772379.944781:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772379.944835:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772379.944871:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772380.072786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772380.072788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772380.136747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772380.200785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772380.264769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772380.264793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772380.264803:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000008:00020000:0.0:1790772380.328798:0:6623:0:(osc_cache.c:999:osc_extent_wait()) extent ffff9c4ec8085cc0@{[0 -> 0/1023], [3|0|+|rpc|wiuY|ffff9c4ec4d7ec40], [28672|1|+|-|ffff9c4efbf60400|1024|ffff9c4ec134cf80]} lustre-OST0000-osc-ffff9c4ec54bd800: wait ext to 0 timedout, recovery in progress? 00000008:00020000:0.0:1790772380.328829:0:6623:0:(osc_cache.c:999:osc_extent_wait()) ### extent: 00000000fb4c9473 ns: lustre-OST0000-osc-ffff9c4ec54bd800 lock: 00000000151847b3/0x61116bf958e324f1 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0xf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800029400000000 nid: local remote: 0xfa0c505d97f0382e expref: -99 pid: 11208 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00020000:3.0:1790772380.328872:0:14200:0:(osc_cache.c:999:osc_extent_wait()) extent ffff9c4ec8085d80@{[0 -> 0/1023], [3|0|+|rpc|wiuY|ffff9c4ec4d7fc00], [28672|1|+|-|ffff9c4efd937800|1024|ffff9c4ec134a7c0]} lustre-OST0000-osc-ffff9c4ee0fe1800: wait ext to 0 timedout, recovery in progress? 00000100:00100000:2.0:1790772380.328879:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000008:00020000:3.0:1790772380.328895:0:14200:0:(osc_cache.c:999:osc_extent_wait()) ### extent: 00000000163b1a30 ns: lustre-OST0000-osc-ffff9c4ee0fe1800 lock: 00000000e25dde2d/0x61116bf958e36342 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800029400020000 nid: local remote: 0xfa0c505d97f0eaca expref: -99 pid: 11733 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00020000:2.0:1790772380.328924:0:11696:0:(osc_cache.c:999:osc_extent_wait()) extent ffff9c4ec6bf3180@{[0 -> 9/1023], [3|0|+|rpc|wiuY|ffff9c4ef33ee540], [65536|10|+|-|ffff9c4ec3161000|1024|ffff9c4ec134cf80]} lustre-OST0001-osc-ffff9c4ec54bd800: wait ext to 0 timedout, recovery in progress? 00000008:00020000:2.0:1790772380.328943:0:11696:0:(osc_cache.c:999:osc_extent_wait()) ### extent: 00000000541fdf14 ns: lustre-OST0001-osc-ffff9c4ec54bd800 lock: 00000000fb748277/0x61116bf958e33226 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x20:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800029400020000 nid: local remote: 0xfa0c505d97f05fbf expref: -99 pid: 14038 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:3.0:1790772380.392779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772380.392792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772380.520762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772380.520779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772380.520809:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772380.584784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772380.584836:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772380.584843:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772380.584926:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772380.584955:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772380.712776:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772380.776768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772380.840771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772380.904778:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772380.904799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772380.904846:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772380.968800:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772380.968812:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772380.969051:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772381.096748:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772381.096774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772381.160747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772381.224756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772381.288743:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772381.288776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772381.288787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772381.352767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772381.416782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772381.416823:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772381.544776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772381.544777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772381.544829:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772381.608806:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772381.608815:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772381.608845:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772381.608885:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772381.608905:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772381.736755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772381.800731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772381.864757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772381.928834:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772381.928854:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772381.928875:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772381.992795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772381.992807:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772382.120736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772382.120771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772382.184814:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772382.248880:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772382.312791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772382.312791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772382.312827:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772382.376747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772382.440772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772382.440794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772382.568765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772382.568813:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772382.568821:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772382.632784:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772382.632790:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772382.632814:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772382.632896:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772382.632908:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772382.760753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772382.824746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772382.888737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772382.952769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772382.952777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772382.952806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772383.016848:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772383.016933:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772383.144754:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772383.144783:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772383.208793:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772383.272844:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772383.336811:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772383.336827:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772383.336841:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772383.400769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772383.464797:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772383.464802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772383.592800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772383.592840:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772383.592899:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772383.656808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772383.656858:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772383.656858:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772383.656911:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772383.784782:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772383.848793:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772383.912716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772383.912941:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772383.912952:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772383.912959:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772383.912972:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1379 00000100:00080000:3.0:1790772383.912985:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772383.913000:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772383.913094:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534438912/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772383.913134:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772383.913140:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772383.913143:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772383.913151:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1379 00000100:00080000:3.0:1790772383.913157:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772383.913167:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772383.913183:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772383.913188:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534438912:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772383.913210:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534439040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772383.913237:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772383.913244:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772383.913249:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772383.913258:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1379 00000100:00080000:3.0:1790772383.913281:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772383.913292:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772383.913299:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790772383.913315:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772383.913347:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534439168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772383.913364:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772383.913373:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1790772383.913375:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772383/real 1790772383] req@ffff9c4ef73c2d80 x1877759534438912/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772383.913386:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772383.913411:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1379 00000100:00080000:3.0:1790772383.913423:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772383.913437:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772383.913439:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534438912/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772383.913450:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772383.913472:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534439296/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772383.913473:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772383.913494:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772383.913513:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772383.913521:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534438912:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772383.913542:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772383.913546:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772383.913552:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1379 00000100:00080000:3.0:1790772383.913556:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772383.913563:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772383.913568:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534439040:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772383.913570:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772383.913626:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534439424/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772383.913632:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772383.913657:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772383.913663:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772383.913666:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772383.913674:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1379 00000100:00100000:2.0:1790772383.913680:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772383/real 1790772383] req@ffff9c4ef73c0e00 x1877759534439040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772383.913699:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534439040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772383.913720:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772383.913724:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772383.913730:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534439040:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772383.913733:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772383.913740:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534439168:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772383.913749:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772383.913760:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790772383.913770:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772383.913786:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534439552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772383.913792:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772383/real 1790772383] req@ffff9c4ef73c0000 x1877759534439168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772383.913848:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772383.913856:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534439168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772383.913857:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772383.913864:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772383.913883:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772383.913884:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1379 00000100:00080000:3.0:1790772383.913902:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772383.913903:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772383.913919:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772383.913920:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534439168:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772383.913935:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772383.913961:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534439680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772383.913968:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534439296:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772383.913985:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772383.913991:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772383.913994:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772383.914002:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1379 00000100:00080000:3.0:1790772383.914013:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772383.914026:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772383.914034:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790772383.914034:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772383.914056:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534439808/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772383.914076:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772383/real 1790772383] req@ffff9c4ef73c1c00 x1877759534439296/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772383.914161:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772383.914172:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534439296/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772383.914172:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772383.914182:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772383.914207:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1379 00000100:00080000:2.0:1790772383.914212:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772383.914218:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772383.914221:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772383.914233:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534439296:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772383.914238:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772383.914257:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772383.914272:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534439424:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772383.914287:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534439936/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772383.914307:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772383.914325:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772383/real 1790772383] req@ffff9c4ef73c2a00 x1877759534439424/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772383.914336:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534439424/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772383.914352:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772383.914356:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772383.914359:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534439424:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772383.914365:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534439552:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772383.914379:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772383.914386:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772383/real 1790772383] req@ffff9c4ef73c0700 x1877759534439552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772383.914394:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534439552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772383.914413:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772383.914416:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772383.914419:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534439552:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772383.914424:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534439680:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772383.914443:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772383.914457:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772383/real 1790772383] req@ffff9c4ef73c2300 x1877759534439680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772383.914471:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534439680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772383.914481:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772383.914485:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772383.914487:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534439680:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772383.914493:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534439808:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772383.914512:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772383.914531:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772383/real 1790772383] req@ffff9c4ef73c0a80 x1877759534439808/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772383.914540:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534439808/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772383.914548:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772383.914550:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772383.914553:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534439808:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772383.914567:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534439936:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772383.914580:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772383.914597:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772383/real 1790772383] req@ffff9c4ef73c1500 x1877759534439936/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772383.914605:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534439936/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772383.914613:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772383.914615:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772383.914617:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534439936:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772383.976749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772383.976750:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772383.976794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772384.040782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772384.040792:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772384.168782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772384.168787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772384.232776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772384.296811:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772384.360791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772384.360791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772384.360842:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772384.424754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772384.488761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772384.488791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772384.616763:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772384.616783:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772384.616868:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772384.680746:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772384.680746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772384.680782:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772384.680868:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772384.681072:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772384.808785:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772384.872787:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772384.936763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772385.000781:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772385.000785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772385.000829:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772385.064756:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772385.064778:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772385.192812:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772385.192813:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772385.256766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772385.320834:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772385.384774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772385.384789:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772385.384809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772385.448734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772385.512785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772385.512800:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772385.640766:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772385.640794:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772385.640803:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772385.704821:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772385.704822:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772385.704840:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772385.704924:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772385.704952:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772385.832748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772385.896764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772385.960773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772386.024775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772386.024791:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772386.024833:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772386.088801:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772386.088842:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772386.216752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772386.216762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772386.280772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772386.344784:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772386.408775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772386.408804:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772386.408813:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772386.472770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772386.536760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772386.536790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772386.664760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772386.664794:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772386.664810:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772386.728832:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772386.728855:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772386.728858:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772386.728905:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772386.856753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772386.920768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772386.984781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772387.048762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772387.048775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772387.048806:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772387.112776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772387.112808:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772387.240770:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772387.240780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772387.304751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772387.368778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772387.432774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772387.432777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772387.432792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772387.496780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772387.560811:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772387.560812:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772387.688758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772387.688786:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772387.688800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772387.752813:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772387.752826:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772387.752887:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772387.880756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772387.944751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772388.008742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772388.072806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772388.072806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772388.072845:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772388.136762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772388.136802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772388.264759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772388.264782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772388.328750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772388.392765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772388.456766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772388.456770:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772388.456818:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772388.520812:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772388.584756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772388.584787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772388.712758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772388.712798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772388.712806:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772388.776756:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772388.776785:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772388.776801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772388.776821:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772388.776856:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772388.904743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772388.968759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772389.032750:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772389.032761:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772389.032769:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772389.032779:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772389.032788:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1384 00000100:00080000:3.0:1790772389.032807:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772389.032828:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772389.032894:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534440064/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772389.032940:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772389.032948:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772389.032953:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772389.032982:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1384 00000100:00080000:3.0:1790772389.032990:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772389.033002:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772389.033025:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772389.033048:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534440064:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772389.033053:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534440192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772389.033074:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772389.033081:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772389.033085:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772389.033093:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1384 00000100:00080000:3.0:1790772389.033099:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772389.033108:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772389.033118:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772389.033155:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534440320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772389.033174:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772389.033182:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772389.033186:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772389.033194:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1384 00000100:00080000:3.0:1790772389.033201:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772389.033211:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772389.033221:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772389.033241:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534440192:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772389.033242:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534440448/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772389.033266:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772389.033273:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772389.033278:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772389.033287:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1384 00000100:00080000:3.0:1790772389.033307:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772389.033320:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772389.033326:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534440320:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772389.033343:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772389.033370:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534440576/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772389.033380:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534440448:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772389.033392:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772389.033399:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772389.033403:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772389.033412:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1384 00000100:00080000:3.0:1790772389.033419:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772389.033427:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534440576:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772389.033429:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772389.033439:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772389.033461:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534440704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772389.033494:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772389.033503:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772389.033508:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772389.033531:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1384 00000100:00100000:2.0:1790772389.033539:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534440704:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772389.033540:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772389.033552:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772389.033563:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772389.033589:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534440832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772389.033629:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772389.033637:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772389.033641:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772389.033644:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534440832:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772389.033689:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1384 00000100:00080000:3.0:1790772389.033697:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772389.033709:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772389.033720:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772389.033746:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534440960/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772389.033776:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772389.033784:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772389.033788:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772389.033797:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1384 00000100:00080000:3.0:1790772389.033804:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772389.033812:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534440960:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772389.033814:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772389.033823:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772389.033844:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534441088/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772389.033877:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534441088:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772389.096799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772389.096799:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772389.096799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772389.160763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772389.160770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772389.288752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772389.288785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772389.352765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772389.416776:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772389.480758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772389.480787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772389.480788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772389.544767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772389.608747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772389.608776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772389.736798:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772389.736815:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772389.736815:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772389.800759:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772389.800807:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772389.800807:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772389.800844:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772389.928763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772389.992747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772390.056762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772390.120748:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772390.120772:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772390.120809:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772390.184770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772390.184789:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772390.312765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772390.312799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772390.376759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772390.440776:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772390.504750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772390.504774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772390.504804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772390.568785:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772390.632778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772390.632791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772390.760772:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772390.760777:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772390.760794:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772390.824766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772390.824781:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772390.824783:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772390.824831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772390.824857:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772390.952741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772391.016746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772391.080775:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772391.144754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772391.144778:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772391.144805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772391.208776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772391.208848:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772391.336778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772391.336782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772391.400760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772391.464768:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772391.528770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772391.528785:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772391.528790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772391.592749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772391.656768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772391.656770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772391.784783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772391.784783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772391.784801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772391.848781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772391.848811:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772391.848817:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772391.848864:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772392.040774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772392.104765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772392.105449:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772389/real 1790772392] req@ffff9c4ef73c0e00 x1877759534441088/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772392.105520:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534441088/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772392.105550:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772392.105561:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772392.105571:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534441088:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772392.105588:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772389/real 1790772392] req@ffff9c4ef73c0000 x1877759534440960/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772392.105616:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534440960/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772392.105702:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772392.105713:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772392.105722:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534440960:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772392.105741:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772389/real 1790772392] req@ffff9c4ef73c1c00 x1877759534440832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772392.105772:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534440832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772392.105789:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772392.105797:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772392.105804:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534440832:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772392.105818:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772389/real 1790772392] req@ffff9c4ef73c2a00 x1877759534440704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772392.105843:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534440704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772392.105874:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772392.105883:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772392.105937:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534440704:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772392.105956:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772389/real 1790772392] req@ffff9c4ef73c0700 x1877759534440576/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772392.105992:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534440576/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772392.106015:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772392.106035:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772392.106043:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534440576:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772392.106062:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772389/real 1790772392] req@ffff9c4ef73c2300 x1877759534440448/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772392.106088:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534440448/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772392.106108:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772392.106115:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772392.106122:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534440448:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772392.106136:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772389/real 1790772392] req@ffff9c4ef73c1500 x1877759534440192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772392.106163:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534440192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772392.106193:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772392.106201:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772392.106209:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534440192:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772392.106222:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772389/real 1790772392] req@ffff9c4ef73c0a80 x1877759534440320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772392.106247:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534440320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772392.106263:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772392.106270:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772392.106277:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534440320:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772392.106293:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772389/real 1790772392] req@ffff9c4ef73c3800 x1877759534440064/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772392.106318:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534440064/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772392.106335:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772392.106341:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772392.106349:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534440064:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772392.168771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772392.168795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772392.168796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772392.232796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772392.232999:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772392.360758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772392.360773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772392.424753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772392.488767:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772392.552740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772392.552813:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772392.552814:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772392.616773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772392.680754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772392.680788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772392.808751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772392.808771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772392.808802:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772392.872792:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772392.872800:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772392.872857:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772392.872922:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772392.872935:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772393.064786:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772393.128775:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772393.192857:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772393.192858:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772393.192862:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772393.256803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772393.257095:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772393.384805:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772393.384822:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772393.448786:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772393.512763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772393.576797:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772393.576805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772393.576822:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772393.640746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772393.704711:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772393.704723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772393.832732:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772393.832732:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772393.832738:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772393.896727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772393.896738:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772393.896740:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772393.896770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772393.896772:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772394.088694:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772394.152788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772394.152789:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772394.152799:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772394.152805:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772394.152840:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1390 00000100:00080000:3.0:1790772394.152905:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772394.152922:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772394.152989:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534441216/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772394.153019:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772394.153022:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772394.153024:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772394.153028:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1390 00000100:00080000:3.0:1790772394.153032:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772394.153038:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772394.153050:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772394.153056:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534441216:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772394.153063:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534441344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772394.153072:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772394.153075:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772394.153077:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772394.153081:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1390 00000100:00080000:3.0:1790772394.153084:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772394.153088:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772394.153092:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772394.153101:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534441472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772394.153143:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772394.153146:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772394.153148:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772394.153151:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1390 00000100:00080000:3.0:1790772394.153163:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772394.153168:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772394.153173:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:2.0:1790772394.153181:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772394.153183:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534441600/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772394.153192:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772394.153195:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772394.153197:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772394.153200:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1390 00000100:00080000:3.0:1790772394.153203:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772394.153208:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772394.153211:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772394.153230:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534441728/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772394.153237:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772394.153241:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772394.153242:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772394.153245:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1390 00000100:00080000:3.0:1790772394.153248:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772394.153256:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772394.153257:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772394/real 1790772394] req@ffff9c4ef73c2d80 x1877759534441216/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772394.153271:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772394.153288:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534441216/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772394.153289:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534441856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772394.153325:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772394.153328:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772394.153331:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772394.153336:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772394.153338:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772394.153345:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1390 00000100:00080000:3.0:1790772394.153353:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772394.153354:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534441216:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772394.153362:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772394.153376:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772394.153391:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534441984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772394.153403:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772394.153404:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534441344:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772394.153412:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772394.153431:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772394.153440:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1390 00000800:00000100:2.0:1790772394.153444:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772394.153447:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772394.153458:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772394.153465:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772394/real 1790772394] req@ffff9c4ef73c3800 x1877759534441344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772394.153466:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772394.153493:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534442112/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772394.153494:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534441344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772394.153522:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772394.153528:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772394.153532:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772394.153535:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772394.153539:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1390 00000100:00080000:2.0:1790772394.153543:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772394.153547:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772394.153551:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534441344:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772394.153558:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772394.153565:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534441472:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772394.153567:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772394.153586:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534442240/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772394.153598:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772394.153616:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772394/real 1790772394] req@ffff9c4ef73c0a80 x1877759534441472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772394.153631:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534441472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772394.153673:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772394.153678:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772394.153682:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534441472:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772394.153690:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534441600:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772394.153715:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772394.153728:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772394/real 1790772394] req@ffff9c4ef73c1500 x1877759534441600/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772394.153741:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534441600/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772394.153752:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772394.153756:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772394.153759:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534441600:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772394.153766:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534441728:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772394.153787:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772394.153800:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772394/real 1790772394] req@ffff9c4ef73c2300 x1877759534441728/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772394.153813:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534441728/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772394.153825:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772394.153828:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772394.153832:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534441728:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772394.153840:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534441856:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772394.153878:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772394.153892:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772394/real 1790772394] req@ffff9c4ef73c0700 x1877759534441856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772394.153908:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534441856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772394.153928:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772394.153932:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772394.153936:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534441856:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772394.153965:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534441984:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772394.153987:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772394.154000:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772394/real 1790772394] req@ffff9c4ef73c2a00 x1877759534441984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772394.154013:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534441984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772394.154023:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772394.154027:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772394.154031:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534441984:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772394.154038:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534442112:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772394.154066:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772394.154079:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772394/real 1790772394] req@ffff9c4ef73c1c00 x1877759534442112/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772394.154093:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534442112/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772394.154106:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772394.154111:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772394.154115:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534442112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772394.154124:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534442240:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772394.154147:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772394.154162:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772394/real 1790772394] req@ffff9c4ef73c0000 x1877759534442240/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772394.154178:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534442240/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772394.154195:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772394.154199:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772394.154202:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534442240:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772394.216790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772394.216795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772394.216828:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772394.280910:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772394.280934:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772394.281017:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772394.408766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772394.408767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772394.472749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772394.536765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772394.600766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772394.600789:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772394.600831:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772394.664742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772394.728801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772394.728801:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772394.856761:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772394.856816:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772394.856817:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772394.920774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772394.920807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772394.920899:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772394.920913:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772394.920914:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772395.112734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772395.176795:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772395.240814:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772395.240834:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772395.240834:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772395.304761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772395.304883:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772395.304994:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772395.432770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772395.432885:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772395.496762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772395.560806:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772395.624760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772395.624771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772395.624800:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772395.688768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772395.752774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772395.752820:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772395.880792:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772395.880817:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772395.880853:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772395.944804:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772395.944859:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772395.944865:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772395.944928:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772395.944936:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772396.136765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772396.200777:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772396.264793:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772396.264802:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772396.264849:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772396.328847:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772396.328938:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772396.329045:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772396.456839:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772396.456915:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772396.520785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772396.584766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772396.648771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772396.648771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772396.648805:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772396.712771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772396.776767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772396.776805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772396.904845:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772396.904846:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772396.904854:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772396.968783:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772396.968847:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772396.968856:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772396.968910:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772396.968932:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772397.160794:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772397.224752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772397.288822:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772397.288831:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772397.288856:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772397.352778:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772397.352845:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772397.352949:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772397.480761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772397.480787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772397.544770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772397.608760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772397.672760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772397.672833:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772397.672833:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772397.736811:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772397.800778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772397.800844:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772397.928826:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772397.928854:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772397.928855:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772397.992771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772397.992848:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772397.992885:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772397.992902:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772398.184812:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772398.248833:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772398.312813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772398.312816:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772398.312837:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772398.376804:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772398.376868:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772398.376969:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772398.504735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772398.504797:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772398.568732:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772398.632746:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772398.696761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772398.696820:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772398.696838:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772398.760797:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772398.824821:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772398.824842:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772398.952796:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772398.952846:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772398.952911:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772399.016816:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772399.016821:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772399.016823:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772399.016920:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772399.016942:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772399.208753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772399.272812:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772399.272826:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772399.272837:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772399.272844:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772399.272860:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1395 00000100:00080000:0.0:1790772399.272877:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772399.272898:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772399.272971:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534442368/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772399.273029:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772399.273035:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772399.273039:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772399.273047:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1395 00000100:00080000:0.0:1790772399.273054:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772399.273064:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772399.273086:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772399.273087:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534442368:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772399.273128:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534442496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772399.273157:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772399.273165:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772399.273171:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772399.273178:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1395 00000100:00080000:0.0:1790772399.273198:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772399.273211:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772399.273223:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772399.273235:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772399.273257:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534442624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772399.273272:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772399.273278:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772399.273284:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772399.273287:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772399/real 1790772399] req@ffff9c4ec7a7f480 x1877759534442368/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772399.273295:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1395 00000100:00080000:0.0:1790772399.273307:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772399.273319:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772399.273329:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772399.273337:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534442368/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772399.273352:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534442752/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772399.273360:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772399.273375:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772399.273377:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772399.273388:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1790772399.273393:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534442368:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772399.273395:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772399.273410:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1395 00000100:00080000:0.0:1790772399.273416:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772399.273422:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772399.273429:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772399.273443:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534442496:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772399.273447:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534442880/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772399.273468:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772399.273473:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772399.273477:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790772399.273488:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772399.273489:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1395 00000100:00080000:0.0:1790772399.273501:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772399.273510:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772399.273520:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772399.273528:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772399/real 1790772399] req@ffff9c4ec7a7ed80 x1877759534442496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772399.273544:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534443008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772399.273558:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534442496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772399.273569:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772399.273583:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772399.273589:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772399.273601:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1395 00000100:00080000:0.0:1790772399.273607:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772399.273616:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772399.273617:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790772399.273625:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772399.273629:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772399.273639:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534442496:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772399.273680:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534443136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772399.273696:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772399.273704:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1790772399.273706:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534442624:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772399.273709:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772399.273726:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1395 00000100:00080000:0.0:1790772399.273737:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772399.273747:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772399.273783:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:2.0:1790772399.273793:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772399.273814:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534443264/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772399.273818:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772399/real 1790772399] req@ffff9c4ec7a7d880 x1877759534442624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772399.273862:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772399.273873:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534442624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772399.273874:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772399.273882:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772399.273899:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1395 00000100:00080000:2.0:1790772399.273905:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772399.273908:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772399.273918:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772399.273928:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772399.273929:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534442624:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772399.273949:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534442752:192.168.204.150@tcp:8:kworker.0 00000008:00100000:0.0:1790772399.273952:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772399.273989:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534443392/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772399.273990:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772399.274011:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772399/real 1790772399] req@ffff9c4ec7a7c380 x1877759534442752/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772399.274035:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534442752/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772399.274053:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772399.274059:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772399.274064:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534442752:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772399.274089:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534442880:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772399.274112:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772399.274126:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772399/real 1790772399] req@ffff9c4ec7a7e300 x1877759534442880/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772399.274141:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534442880/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772399.274154:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772399.274158:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772399.274162:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534442880:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772399.274170:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534443008:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772399.274190:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772399.274202:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772399/real 1790772399] req@ffff9c4ec7a7ca80 x1877759534443008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772399.274217:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534443008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772399.274238:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772399.274242:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772399.274246:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534443008:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772399.274254:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534443136:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772399.274273:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772399.274285:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772399/real 1790772399] req@ffff9c4ec7a7c700 x1877759534443136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772399.274300:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534443136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772399.274312:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772399.274317:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772399.274321:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534443136:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772399.274328:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534443264:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772399.274348:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772399.274359:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772399/real 1790772399] req@ffff9c4ec7a7fb80 x1877759534443264/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772399.274373:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534443264/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772399.274387:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772399.274391:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772399.274396:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534443264:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772399.274403:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534443392:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772399.274423:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772399.274434:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772399/real 1790772399] req@ffff9c4ec7a7d180 x1877759534443392/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772399.274448:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534443392/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772399.274461:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772399.274465:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772399.274471:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534443392:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772399.336828:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772399.336841:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772399.336858:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772399.400863:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772399.400918:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772399.528806:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772399.528832:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772399.592751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772399.656834:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772399.720795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772399.720822:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772399.720832:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772399.784804:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772399.848831:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772399.848851:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772399.976784:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772399.976784:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772399.976876:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772400.040874:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772400.040899:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772400.040931:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772400.040993:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772400.232836:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772400.296808:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772400.360769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772400.360811:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772400.360824:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772400.424795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772400.424876:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772400.552830:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772400.552861:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772400.616785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772400.680819:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772400.744752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772400.744801:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772400.744808:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772400.808766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772400.872807:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772400.872807:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772401.000830:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772401.000860:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772401.064850:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772401.064872:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772401.064887:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772401.064940:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772401.064951:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772401.256792:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772401.320788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772401.384845:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772401.384845:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772401.384847:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772401.448878:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772401.448966:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772401.448978:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772401.576767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772401.576808:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772401.640810:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772401.704813:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772401.768826:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772401.768828:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772401.768846:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772401.832803:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772401.896831:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772401.896904:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772402.024832:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772402.024859:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772402.024940:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772402.088796:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772402.088888:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772402.088888:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772402.088950:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772402.280802:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772402.344771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772402.408780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772402.408786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772402.408842:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772402.472812:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772402.472934:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772402.473054:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772402.600765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772402.600809:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772402.664789:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772402.728855:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772402.792796:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772402.792809:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772402.792850:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772402.856806:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772402.920782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772402.920844:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772403.048860:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772403.048863:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772403.048888:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772403.112788:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772403.112808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772403.112841:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772403.112855:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772403.112892:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772403.304767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772403.368786:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772403.432794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772403.432796:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772403.432845:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772403.496904:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772403.496947:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772403.497048:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772403.624771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772403.624805:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772403.688777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772403.752881:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772403.816794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772403.816804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772403.816828:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772403.880794:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772403.944808:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772403.944825:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772404.072799:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772404.072820:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772404.072882:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772404.136834:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772404.136858:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772404.136867:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772404.136947:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772404.328778:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772404.392808:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772404.392813:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772404.392819:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772404.392831:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772404.392849:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1400 00000100:00080000:0.0:1790772404.392868:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772404.392888:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772404.392954:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534443520/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772404.392995:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772404.393002:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772404.393006:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772404.393013:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1400 00000100:00080000:0.0:1790772404.393022:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772404.393031:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772404.393048:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772404.393067:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534443520:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772404.393071:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534443648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772404.393093:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772404.393099:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772404.393101:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772404.393108:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1400 00000100:00080000:0.0:1790772404.393113:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772404.393119:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772404.393126:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772404.393142:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534443776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772404.393154:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772404.393158:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772404.393161:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772404.393168:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1400 00000100:00080000:0.0:1790772404.393174:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772404.393180:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772404.393187:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:2.0:1790772404.393188:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772404.393208:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534443904/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772404.393224:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772404.393230:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772404.393233:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772404.393238:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1400 00000100:00080000:0.0:1790772404.393244:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772404.393251:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772404.393258:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772404.393269:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772404/real 1790772404] req@ffff9c4ec7a7df80 x1877759534443520/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772404.393277:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534444032/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772404.393328:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772404.393333:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772404.393336:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772404.393342:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534443520/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772404.393348:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1400 00000100:00080000:0.0:1790772404.393358:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772404.393371:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772404.393376:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790772404.393383:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772404.393391:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772404.393407:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534443520:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772404.393408:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534444160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772404.393427:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772404.393433:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772404.393437:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772404.393444:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1400 00000100:00080000:0.0:1790772404.393450:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772404.393456:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534443648:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772404.393462:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772404.393473:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772404.393492:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534444288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772404.393507:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772404.393508:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772404.393517:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772404.393520:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772404.393528:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1400 00000100:00080000:0.0:1790772404.393534:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772404.393539:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772404/real 1790772404] req@ffff9c4ec7a7d180 x1877759534443648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772404.393546:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772404.393558:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772404.393577:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534443648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772404.393582:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534444416/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772404.393632:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772404.393639:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772404.393643:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772404.393649:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1400 00000100:00080000:2.0:1790772404.393727:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772404.393732:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772404.393746:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772404.393747:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772404.393763:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772404.393771:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534443648:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772404.393787:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534444544/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772404.393797:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534443776:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772404.393859:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772404.393886:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772404/real 1790772404] req@ffff9c4ec7a7fb80 x1877759534443776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772404.393914:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534443776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772404.393933:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772404.393941:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772404.393949:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534443776:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772404.393962:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534443904:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772404.394008:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772404.394032:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772404/real 1790772404] req@ffff9c4ec7a7c700 x1877759534443904/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772404.394049:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534443904/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772404.394064:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772404.394068:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772404.394074:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534443904:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772404.394082:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534444032:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772404.394108:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772404.394120:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772404/real 1790772404] req@ffff9c4ec7a7ca80 x1877759534444032/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772404.394135:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534444032/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772404.394148:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772404.394152:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772404.394156:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534444032:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772404.394163:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534444160:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772404.394183:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772404.394195:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772404/real 1790772404] req@ffff9c4ec7a7e300 x1877759534444160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772404.394210:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534444160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772404.394232:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772404.394237:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772404.394241:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534444160:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772404.394282:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534444288:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772404.394309:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772404.394329:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772404/real 1790772404] req@ffff9c4ec7a7c380 x1877759534444288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772404.394344:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534444288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772404.394361:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772404.394369:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772404.394378:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534444288:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772404.394391:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534444416:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772404.394433:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772404.394457:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772404/real 1790772404] req@ffff9c4ec7a7d880 x1877759534444416/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772404.394486:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534444416/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772404.394504:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772404.394510:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772404.394516:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534444416:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772404.394524:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534444544:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772404.394555:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772404.394570:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772404/real 1790772404] req@ffff9c4ec7a7ed80 x1877759534444544/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772404.394586:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534444544/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772404.394599:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772404.394603:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772404.394608:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534444544:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772404.456801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772404.456802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772404.456806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772404.520813:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772404.520888:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772404.521007:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772404.648812:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772404.648813:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772404.712795:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772404.776785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772404.840809:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772404.840818:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772404.840825:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772404.904776:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772404.968794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772404.968814:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772405.096841:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772405.096856:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772405.096857:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772405.160750:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772405.160842:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772405.160892:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772405.160895:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772405.160907:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772405.352771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772405.416748:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772405.480798:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772405.480798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772405.480802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772405.544814:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772405.544851:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772405.672782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772405.672792:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772405.736772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772405.800770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772405.864790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772405.864808:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772405.864819:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772405.928764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772405.992774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772405.992782:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772406.120791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772406.120795:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772406.120810:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772406.184779:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772406.184836:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772406.184842:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772406.184899:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772406.312820:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772406.376762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772406.441083:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772406.504792:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772406.504796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772406.504798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772406.568743:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772406.568830:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772406.696758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772406.696812:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772406.760760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772406.824770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772406.888821:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772406.888821:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772406.888822:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772406.952753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772407.016769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772407.016806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772407.144770:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772407.144770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772407.144808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772407.208807:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772407.208819:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772407.208824:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772407.208877:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772407.208883:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772407.336757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772407.400754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772407.464817:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772407.528761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772407.528822:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772407.528855:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772407.592868:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772407.592957:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772407.592999:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772407.720739:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772407.720788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772407.784759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772407.848758:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772407.912790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772407.912791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772407.912850:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772407.976761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772408.040773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772408.040779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772408.168802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772408.168802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772408.168844:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772408.232800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772408.232810:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772408.232815:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772408.232851:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772408.232863:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772408.360788:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772408.424814:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772408.488781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772408.552763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772408.552802:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772408.552816:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772408.616802:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772408.616848:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772408.616938:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772408.744748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772408.744789:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772408.808769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772408.872786:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772408.936731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772408.936782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772408.936791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772409.000767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772409.064772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772409.064791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772409.192776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772409.192780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772409.192787:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772409.256758:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772409.256790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772409.256793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772409.256839:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772409.384743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772409.448753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772409.512799:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772409.512804:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772409.512822:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772409.512833:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772409.512862:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1405 00000100:00080000:0.0:1790772409.512890:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772409.512910:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772409.512985:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534444672/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772409.513060:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772409.513066:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772409.513070:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772409.513086:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1405 00000100:00080000:0.0:1790772409.513094:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772409.513109:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772409.513126:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772409.513146:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534444672:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772409.513168:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534444800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772409.513209:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772409.513218:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772409.513224:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772409.513234:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1405 00000100:00080000:0.0:1790772409.513244:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772409.513256:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772409.513265:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790772409.513290:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772409.513298:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534444928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772409.513324:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772409.513333:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772409.513339:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772409.513347:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1405 00000100:00080000:0.0:1790772409.513357:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772409.513368:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772409.513378:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772409.513400:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772409/real 1790772409] req@ffff9c4ec7a7f480 x1877759534444672/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772409.513404:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534445056/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772409.513458:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772409.513466:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534444672/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772409.513467:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772409.513473:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772409.513487:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1405 00000100:00080000:0.0:1790772409.513500:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772409.513506:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772409.513512:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772409.513520:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772409.513528:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534444672:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790772409.513531:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772409.513595:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534445184/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772409.513612:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534444800:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772409.513629:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772409.513640:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772409.513644:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772409.513704:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1405 00000100:00080000:0.0:1790772409.513715:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772409.513725:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772409.513735:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772409.513751:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772409.513759:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772409/real 1790772409] req@ffff9c4ec7a7ed80 x1877759534444800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772409.513792:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534445312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772409.513793:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534444800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772409.513862:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772409.513868:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772409.513872:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772409.513878:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772409.513882:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1405 00000100:00080000:0.0:1790772409.513894:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772409.513902:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772409.513911:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772409.513922:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772409.513935:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534445440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772409.513944:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534444800:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772409.513974:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534444928:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772409.514005:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772409.514014:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772409.514020:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790772409.514033:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772409.514034:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1405 00000100:00080000:0.0:1790772409.514048:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772409.514061:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772409.514072:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772409.514083:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772409/real 1790772409] req@ffff9c4ec7a7d880 x1877759534444928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772409.514100:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534445568/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772409.514138:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772409.514150:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772409.514155:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772409.514162:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534444928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772409.514172:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1405 00000100:00080000:0.0:1790772409.514190:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772409.514193:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772409.514206:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772409.514210:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772409.514219:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534444928:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790772409.514228:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772409.514239:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534445056:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772409.514259:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534445696/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772409.514302:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772409.514332:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772409/real 1790772409] req@ffff9c4ec7a7c380 x1877759534445056/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772409.514351:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534445056/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772409.514366:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772409.514371:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772409.514376:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534445056:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772409.514406:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534445184:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772409.514435:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772409.514449:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772409/real 1790772409] req@ffff9c4ec7a7e300 x1877759534445184/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772409.514464:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534445184/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772409.514486:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772409.514491:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772409.514496:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534445184:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772409.514503:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534445312:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772409.514523:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772409.514535:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772409/real 1790772409] req@ffff9c4ec7a7ca80 x1877759534445312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772409.514550:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534445312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772409.514607:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772409.514616:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772409.514623:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534445312:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772409.514637:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534445440:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772409.514726:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772409.514747:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772409/real 1790772409] req@ffff9c4ec7a7c700 x1877759534445440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772409.514763:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534445440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772409.514775:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772409.514779:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772409.514785:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534445440:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772409.514794:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534445568:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772409.514818:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772409.514830:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772409/real 1790772409] req@ffff9c4ec7a7fb80 x1877759534445568/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772409.514845:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534445568/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772409.514859:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772409.514863:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772409.514869:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534445568:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772409.514881:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534445696:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772409.514921:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772409.514946:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772409/real 1790772409] req@ffff9c4ec7a7d180 x1877759534445696/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772409.514972:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534445696/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772409.514988:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772409.514994:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772409.514999:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534445696:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772409.576765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772409.576777:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772409.576804:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772409.640766:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772409.640799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772409.640911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772409.768777:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772409.768787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772409.832775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772409.896741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772409.960744:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772409.960773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772409.960774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772410.024786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772410.088803:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772410.088808:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772410.216772:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772410.216777:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772410.216817:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772410.280781:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772410.280782:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772410.280785:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772410.280846:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772410.408754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772410.472769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772410.536765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772410.600786:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772410.600794:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772410.600890:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772410.664775:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772410.664888:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772410.664999:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772410.792781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772410.792823:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772410.856754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772410.920881:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772410.984779:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772410.984798:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772410.984850:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772411.048795:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772411.112778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772411.112800:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772411.240792:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772411.240800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772411.240811:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772411.304791:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772411.304818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772411.304834:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772411.304865:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772411.304913:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772411.432749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772411.496760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772411.560769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772411.624794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772411.624867:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772411.624871:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772411.688828:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772411.688872:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772411.688937:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772411.816757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772411.816768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772411.880784:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772411.944792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772412.008772:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772412.008773:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772412.008782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772412.072801:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772412.136802:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772412.136809:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772412.264768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772412.264776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772412.264808:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772412.328790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772412.328800:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772412.328811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772412.328857:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772412.328871:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772412.456787:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772412.520750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772412.584768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772412.648814:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772412.648819:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772412.648856:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772412.712837:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772412.712916:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772412.713099:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772412.840760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772412.840788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772412.904752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772412.968797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772413.032752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772413.032759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772413.032797:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772413.096727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772413.160780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772413.160807:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772413.288768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772413.288793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772413.288808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772413.352766:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772413.352796:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772413.352854:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772413.352856:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772413.352866:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772413.480774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772413.544770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772413.608760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772413.672804:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772413.672811:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772413.672817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772413.736826:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772413.736829:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772413.736931:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772413.864776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772413.864785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772413.928746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772413.992794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772414.056783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772414.056784:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772414.056787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772414.120758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772414.184778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772414.184804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772414.312766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772414.312791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772414.312801:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772414.376792:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772414.376804:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772414.376813:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772414.376860:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772414.376875:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772414.504753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772414.568772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790772414.632823:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772414.632835:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772414.632844:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772414.632864:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1410 00000100:00100000:3.0:1790772414.632866:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772414.632884:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772414.632907:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772414.632974:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534445824/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772414.633016:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772414.633023:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772414.633026:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772414.633032:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1410 00000100:00080000:2.0:1790772414.633038:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772414.633046:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772414.633066:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772414.633072:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534445824:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772414.633102:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534445952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772414.633129:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772414.633138:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772414.633144:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772414.633154:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1410 00000100:00080000:2.0:1790772414.633163:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772414.633175:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772414.633187:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1790772414.633187:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772414.633216:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534446080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772414.633232:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772414.633238:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772414.633241:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772414.633249:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1410 00000100:00080000:2.0:1790772414.633257:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772414.633265:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772414.633276:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772414.633278:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772414/real 1790772414] req@ffff9c4efbcc2a00 x1877759534445824/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772414.633299:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534446208/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772414.633314:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772414.633317:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534445824/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772414.633323:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772414.633330:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772414.633346:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1410 00000100:00080000:2.0:1790772414.633355:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772414.633357:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772414.633365:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772414.633374:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790772414.633376:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772414.633388:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534445824:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772414.633399:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534446336/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772414.633414:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772414.633419:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772414.633422:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772414.633428:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1410 00000100:00080000:2.0:1790772414.633433:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772414.633440:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534445952:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772414.633445:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772414.633458:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772414.633513:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534446464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772414.633525:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772414.633546:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772414.633552:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1790772414.633558:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772414/real 1790772414] req@ffff9c4efbcc1500 x1877759534445952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772414.633560:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772414.633577:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1410 00000100:00080000:2.0:1790772414.633590:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772414.633602:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534445952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772414.633605:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772414.633621:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772414.633649:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534446592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772414.633703:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772414.633709:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772414.633712:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772414.633721:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1410 00000100:00080000:3.0:1790772414.633725:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772414.633726:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772414.633743:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772414.633745:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790772414.633759:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772414.633761:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534445952:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772414.633785:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534446080:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772414.633786:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534446720/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772414.633807:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772414.633812:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772414.633815:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772414.633823:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1410 00000100:00080000:2.0:1790772414.633829:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1790772414.633838:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772414.633839:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772414.633852:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772414.633867:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772414/real 1790772414] req@ffff9c4efbcc0380 x1877759534446080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772414.633876:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534446848/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772414.633910:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534446080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772414.633925:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772414.633932:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772414.633938:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534446080:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772414.633950:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534446208:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772414.633975:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772414.633988:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772414/real 1790772414] req@ffff9c4efbcc3100 x1877759534446208/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772414.634003:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534446208/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772414.634017:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772414.634020:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772414.634025:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534446208:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772414.634032:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534446336:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772414.634053:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772414.634065:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772414/real 1790772414] req@ffff9c4efbcc0a80 x1877759534446336/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772414.634079:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534446336/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772414.634091:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772414.634095:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772414.634100:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534446336:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772414.634129:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534446464:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772414.634152:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772414.634165:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772414/real 1790772414] req@ffff9c4efbcc0700 x1877759534446464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772414.634182:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534446464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772414.634204:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772414.634208:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772414.634212:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534446464:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772414.634219:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534446592:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772414.634239:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772414.634252:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772414/real 1790772414] req@ffff9c4efbcc1f80 x1877759534446592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772414.634267:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534446592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772414.634278:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772414.634282:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772414.634286:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534446592:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772414.634294:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534446720:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772414.634313:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772414.634324:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772414/real 1790772414] req@ffff9c4efbcc1880 x1877759534446720/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772414.634340:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534446720/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772414.634363:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772414.634368:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772414.634373:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534446720:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772414.634380:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534446848:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772414.634399:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772414.634409:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772414/real 1790772414] req@ffff9c4efbcc2300 x1877759534446848/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772414.634424:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534446848/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772414.634437:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772414.634441:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772414.634445:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534446848:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772414.696780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772414.696801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772414.696832:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772414.760775:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772414.760795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772414.760942:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772414.888785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772414.888785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772414.952776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772415.016776:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772415.080829:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772415.080829:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772415.080832:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772415.144766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772415.208757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772415.208810:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772415.336938:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772415.337181:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772415.337742:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772415.400822:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772415.400823:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772415.400826:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772415.400914:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772415.528759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772415.592756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772415.656760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772415.720801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772415.720821:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772415.720875:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772415.784799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772415.784861:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772415.784886:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772415.912812:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772415.912812:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772415.976770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772416.040752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772416.104773:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772416.104779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772416.104806:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772416.168768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772416.232769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772416.232781:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772416.360786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772416.360788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772416.360869:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772416.424787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772416.424799:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772416.424860:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772416.424865:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772416.424891:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772416.552753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772416.616747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772416.680788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772416.744820:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772416.744822:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772416.744823:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772416.808787:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772416.808835:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772416.808855:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772416.936763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772416.936807:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772417.000750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772417.064793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772417.128763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772417.128788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772417.128789:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772417.192784:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772417.256824:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772417.256825:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772417.384806:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772417.384957:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772417.385066:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772417.448799:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772417.448844:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772417.448850:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772417.448890:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772417.448901:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772417.448916:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772417.576740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772417.640756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772417.704766:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772417.768776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772417.768826:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772417.768833:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772417.832768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772417.832838:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772417.833000:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772417.960749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772417.960778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772418.024754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772418.088888:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772418.152755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772418.152779:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772418.152804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772418.216752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772418.280782:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772418.280783:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772418.408845:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772418.408889:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772418.408913:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772418.472778:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772418.472812:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772418.472882:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772418.472913:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772418.600729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772418.664739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772418.728760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772418.792813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772418.792865:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772418.792891:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772418.856892:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772418.856893:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772418.857031:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772418.984753:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772418.984771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772419.048754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772419.112781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772419.176758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772419.176882:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772419.176885:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772419.240749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772419.304785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772419.304786:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772419.432812:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772419.432986:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772419.433063:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772419.496771:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772419.496879:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772419.496900:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772419.496936:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772419.497009:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772419.624755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772419.688755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772419.752771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772419.753065:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534446976:192.168.204.150@tcp:250:kworker.0 00000800:00000100:3.0:1790772419.753207:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772419.753297:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772419/real 1790772419] req@ffff9c4efac4df80 x1877759534446976/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772419.753413:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534446976/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772419.753455:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772419.753475:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772419.753492:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534446976:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772419.753551:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534447104:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772419.753610:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772419.753638:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772419/real 1790772419] req@ffff9c4efac4dc00 x1877759534447104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772419.753694:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534447104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772419.753725:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772419.753737:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772419.753757:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534447104:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772419.753773:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534447232:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772419.753821:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772419.753843:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772419/real 1790772419] req@ffff9c4efac4f800 x1877759534447232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772419.753919:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534447232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772419.753951:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772419.753965:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772419.753974:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534447232:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772419.753993:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534447360:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772419.754031:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772419.754046:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772419/real 1790772419] req@ffff9c4efac4f480 x1877759534447360/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772419.754063:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534447360/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772419.754077:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772419.754081:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772419.754085:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534447360:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772419.754093:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534447488:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772419.754113:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772419.754125:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772419/real 1790772419] req@ffff9c4efac4e300 x1877759534447488/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772419.754140:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534447488/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772419.754172:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772419.754179:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772419.754186:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534447488:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772419.754222:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534447616:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772419.754264:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772419.754283:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772419/real 1790772419] req@ffff9c4efac4c700 x1877759534447616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772419.754300:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534447616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772419.754323:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772419.754328:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772419.754333:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534447616:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772419.754341:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534447744:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772419.754394:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772419.754419:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772419/real 1790772419] req@ffff9c4efac4ed80 x1877759534447744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772419.754446:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534447744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772419.754465:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772419.754473:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772419.754481:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534447744:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772419.754491:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534447872:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772419.754521:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772419.754537:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772419/real 1790772419] req@ffff9c4efac4ea00 x1877759534447872/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772419.754553:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534447872/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772419.754568:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772419.754573:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772419.754577:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534447872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772419.754585:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534448000:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772419.754605:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772419.754616:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772419/real 1790772419] req@ffff9c4efac4e680 x1877759534448000/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772419.754631:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534448000/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772419.754644:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772419.754647:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772419.754684:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534448000:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772419.816784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772419.816812:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772419.816829:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772419.880866:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772419.880887:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772419.880928:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772420.008779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772420.008791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772420.072778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772420.136778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772420.200762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772420.200775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772420.200788:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772420.264759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772420.328764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772420.328773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772420.456811:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772420.456915:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772420.457132:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772420.520758:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772420.520778:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772420.520812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772420.520844:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772420.520846:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772420.648749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772420.712748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772420.776733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772420.840780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772420.840780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772420.840789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772420.904760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772420.904835:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772420.904884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772421.032796:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772421.032798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772421.096797:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772421.160784:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772421.224750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772421.224783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772421.224784:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772421.288751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772421.352781:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772421.352831:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772421.480774:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772421.480800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772421.480802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772421.544783:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772421.544790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772421.544808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772421.544856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772421.672748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772421.736760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772421.800756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772421.864782:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772421.864784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772421.864862:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772421.928769:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772421.928812:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772421.928965:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772422.056774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772422.056775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772422.120771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772422.184758:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772422.248784:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772422.248791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772422.248792:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772422.312766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772422.376825:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772422.376836:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772422.504816:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772422.504829:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772422.504836:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772422.568775:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772422.568796:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772422.568835:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772422.568845:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772422.568891:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772422.696751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772422.760737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772422.824756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772422.888932:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772422.888940:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772422.888942:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772422.952830:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772422.952831:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772422.952867:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772423.080764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772423.080773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772423.144771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772423.208770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772423.272759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772423.272781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772423.272800:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772423.336748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772423.400795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772423.400799:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772423.528786:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772423.528789:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772423.528801:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772423.592813:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772423.592828:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772423.592828:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772423.592889:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772423.720778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772423.784760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772423.848754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772423.912843:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772423.912848:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772423.912849:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772423.976850:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772423.976851:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772423.976958:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772424.104761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772424.104767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772424.168748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772424.232757:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772424.296774:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772424.296774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772424.296777:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772424.360768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772424.424780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772424.424853:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772424.552752:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772424.552768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772424.552771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772424.616758:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772424.616781:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772424.616810:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772424.616835:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772424.616867:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772424.744769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772424.808747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772424.872788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772424.872847:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772424.872857:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772424.872864:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772424.872879:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1420 00000100:00080000:0.0:1790772424.872897:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772424.872920:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772424.872996:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534448128/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772424.873051:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772424.873057:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772424.873061:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772424.873067:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1420 00000100:00080000:0.0:1790772424.873075:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772424.873086:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772424.873103:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772424.873111:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534448128:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772424.873133:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534448256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772424.873150:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772424.873155:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772424.873159:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772424.873166:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1420 00000100:00080000:0.0:1790772424.873171:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772424.873178:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772424.873185:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772424.873202:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534448384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772424.873216:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772424.873222:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772424.873225:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1790772424.873230:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772424.873238:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1420 00000100:00080000:0.0:1790772424.873250:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772424.873289:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772424.873301:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772424.873331:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534448512/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772424.873339:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772424/real 1790772424] req@ffff9c4ec7a7df80 x1877759534448128/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772424.873381:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772424.873388:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772424.873391:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772424.873395:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534448128/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772424.873403:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1420 00000100:00080000:0.0:1790772424.873417:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772424.873426:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772424.873430:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772424.873439:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790772424.873444:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772424.873449:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534448128:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772424.873471:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534448640/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772424.873487:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772424.873496:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534448256:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772424.873497:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772424.873504:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772424.873515:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1420 00000100:00080000:0.0:1790772424.873520:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772424.873529:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772424.873538:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1790772424.873545:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772424.873559:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534448768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772424.873571:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772424/real 1790772424] req@ffff9c4ec7a7d180 x1877759534448256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772424.873582:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772424.873595:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772424.873600:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772424.873611:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1420 00000100:00100000:3.0:1790772424.873618:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534448256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772424.873620:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772424.873633:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772424.873644:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772424.873708:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534448896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772424.873727:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772424.873733:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772424.873734:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772424.873742:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772424.873749:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772424.873757:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1420 00000100:00100000:3.0:1790772424.873765:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534448256:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772424.873766:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772424.873780:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772424.873785:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534448384:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790772424.873792:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772424.873814:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534449024/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772424.873828:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1790772424.873832:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772424.873837:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772424.873843:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772424.873851:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1420 00000100:00080000:0.0:1790772424.873857:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772424.873859:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772424/real 1790772424] req@ffff9c4ec7a7fb80 x1877759534448384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772424.873869:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772424.873882:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772424.873895:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534448384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772424.873904:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534449152/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772424.873918:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772424.873934:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772424.873947:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534448384:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772424.873960:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534448512:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772424.873988:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772424.874002:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772424/real 1790772424] req@ffff9c4ec7a7c700 x1877759534448512/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772424.874017:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534448512/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772424.874031:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772424.874035:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772424.874041:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534448512:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772424.874048:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534448640:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772424.874068:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772424.874080:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772424/real 1790772424] req@ffff9c4ec7a7ca80 x1877759534448640/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772424.874094:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534448640/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772424.874107:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772424.874111:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772424.874115:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534448640:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772424.874136:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534448768:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772424.874158:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772424.874171:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772424/real 1790772424] req@ffff9c4ec7a7e300 x1877759534448768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772424.874186:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534448768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772424.874207:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772424.874211:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772424.874215:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534448768:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772424.874223:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534448896:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772424.874242:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772424.874255:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772424/real 1790772424] req@ffff9c4ec7a7c380 x1877759534448896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772424.874293:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534448896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772424.874305:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772424.874310:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772424.874315:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534448896:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772424.874322:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534449024:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772424.874346:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772424.874358:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772424/real 1790772424] req@ffff9c4ec7a7d880 x1877759534449024/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772424.874373:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534449024/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772424.874387:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772424.874391:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772424.874395:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534449024:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772424.874402:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534449152:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772424.874422:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772424.874433:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772424/real 1790772424] req@ffff9c4ec7a7ed80 x1877759534449152/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772424.874449:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534449152/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772424.874462:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772424.874467:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772424.874471:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534449152:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772424.936800:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772424.936818:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772424.936820:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772425.000768:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772425.000812:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772425.000956:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772425.128796:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772425.128820:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772425.192758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772425.256894:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772425.320759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772425.320810:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772425.320812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772425.384752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772425.448777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772425.448800:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772425.576804:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772425.576806:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772425.576807:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772425.640790:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772425.640797:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772425.640822:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772425.640859:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772425.640902:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772425.768739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772425.832771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772425.896767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772425.960761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772425.960798:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772425.960804:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772426.024795:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772426.024812:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772426.152796:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772426.152797:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772426.216738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772426.280820:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772426.344757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772426.344775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772426.344812:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772426.408743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772426.472781:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772426.472821:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772426.600798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772426.600846:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772426.600877:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772426.664809:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772426.664809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772426.664818:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772426.664862:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772426.664874:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772426.792749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772426.856753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772426.920765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772426.984776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772426.984794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772426.984800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772427.048794:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772427.048804:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772427.176771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772427.176785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772427.240755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772427.304785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772427.368771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772427.368772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772427.368816:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772427.432781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772427.496791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772427.496798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772427.624807:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772427.624808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772427.624816:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772427.688809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772427.688810:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772427.688814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772427.688880:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772427.688884:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772427.816749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772427.880768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772427.944749:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772428.008765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772428.008799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772428.008806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772428.072788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772428.072806:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772428.200758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772428.200788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772428.264750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772428.328760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772428.392766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772428.392767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772428.392789:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772428.456764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772428.520758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772428.520801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772428.648764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772428.648808:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772428.648817:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772428.712812:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772428.712812:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772428.712845:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772428.712858:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772428.712858:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772428.840733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772428.904770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772428.968745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772429.032705:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772429.032719:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772429.032725:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772429.096737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772429.096755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772429.224716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772429.224716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772429.352720:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772429.416722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772429.416731:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772429.416737:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772429.480732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772429.544717:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772429.544731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772429.672719:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772429.672720:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772429.672734:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772429.736776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772429.736790:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772429.864700:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772429.928718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772429.992736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772429.992887:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534449280:192.168.204.150@tcp:250:kworker.0 00000800:00000100:3.0:1790772429.992968:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772429.993078:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772429/real 1790772429] req@ffff9c4efac4fb80 x1877759534449280/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772429.993129:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534449280/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772429.993201:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772429.993210:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772429.993218:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534449280:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772429.993267:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534449408:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772429.993304:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772429.993320:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772429/real 1790772429] req@ffff9c4efac4e680 x1877759534449408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772429.993341:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534449408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772429.993363:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772429.993368:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772429.993371:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534449408:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772429.993377:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534449536:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772429.993391:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772429.993398:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772429/real 1790772429] req@ffff9c4efac4ea00 x1877759534449536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772429.993406:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534449536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772429.993413:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772429.993415:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772429.993417:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534449536:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772429.993421:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534449664:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772429.993433:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772429.993439:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772429/real 1790772429] req@ffff9c4efac4ed80 x1877759534449664/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772429.993453:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534449664/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772429.993461:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772429.993463:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772429.993465:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534449664:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772429.993469:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534449792:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772429.993480:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772429.993501:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772429/real 1790772429] req@ffff9c4efac4c700 x1877759534449792/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772429.993509:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534449792/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772429.993517:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772429.993519:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772429.993521:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534449792:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772429.993525:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534449920:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772429.993537:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772429.993543:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772429/real 1790772429] req@ffff9c4efac4e300 x1877759534449920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772429.993567:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534449920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772429.993585:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772429.993588:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772429.993590:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534449920:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772429.993594:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534450048:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772429.993607:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772429.993629:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772429/real 1790772429] req@ffff9c4efac4f480 x1877759534450048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772429.993644:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534450048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772429.993655:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772429.993660:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772429.993664:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534450048:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772429.993704:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534450176:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772429.993716:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772429.993722:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772429/real 1790772429] req@ffff9c4efac4f800 x1877759534450176/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772429.993748:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534450176/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772429.993757:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772429.993760:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772429.993763:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534450176:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772429.993768:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534450304:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772429.993780:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772429.993787:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772429/real 1790772429] req@ffff9c4efac4dc00 x1877759534450304/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772429.993795:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534450304/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772429.993810:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772429.993812:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772429.993814:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534450304:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772430.056740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772430.056759:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772430.056764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772430.120719:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772430.120722:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772430.248786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772430.248843:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772430.376722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772430.440738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772430.440749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772430.440802:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772430.504769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772430.568786:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772430.568804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772430.696776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772430.696841:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772430.696872:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772430.760780:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772430.760795:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772430.760844:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772430.760862:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772430.760873:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772430.888784:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772430.952766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772431.016742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772431.080779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772431.080796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772431.080815:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772431.144791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772431.144799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772431.272778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772431.272788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772431.400765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772431.464762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772431.464770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772431.465717:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772431.528765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772431.592773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772431.592805:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772431.720826:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772431.720868:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772431.720893:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772431.784804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772431.784807:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772431.784817:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772431.784873:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772431.912772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772431.976772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772432.040780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772432.104751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772432.104809:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772432.104819:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772432.168811:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772432.168860:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772432.296785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772432.296794:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772432.424892:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772432.488783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772432.488799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772432.488816:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772432.552799:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772432.616809:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772432.616810:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772432.744831:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772432.744848:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772432.808784:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772432.808806:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772432.808818:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772432.808859:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772432.808884:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772432.936717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772433.000736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772433.064754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772433.128758:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772433.128783:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772433.128785:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772433.192772:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772433.192804:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772433.192872:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772433.320781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772433.320798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772433.448727:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772433.512718:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772433.512718:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772433.512724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772433.576721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772433.640736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772433.640737:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772433.768712:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772433.768716:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772433.768757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772433.832730:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772433.832733:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772433.832733:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772433.832781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772433.832781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772433.961741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772434.024780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772434.088755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772434.152787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772434.152809:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772434.152812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772434.216786:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772434.216820:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772434.216979:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772434.344746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772434.344776:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772434.472723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772434.536794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772434.536794:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772434.536794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772434.600748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772434.664752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772434.664767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772434.792759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772434.792760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772434.792761:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772434.856752:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772434.856782:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772434.856792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772434.856811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772434.856851:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772434.856864:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772434.984730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772435.048781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790772435.112849:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772435.112849:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772435.112863:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772435.112873:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772435.112892:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1431 00000100:00080000:2.0:1790772435.112913:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772435.112939:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772435.113007:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534450432/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772435.113102:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772435.113111:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772435.113115:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772435.113124:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1431 00000100:00080000:2.0:1790772435.113134:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772435.113145:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772435.113155:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534450432:192.168.204.150@tcp:250:kworker.0 00000008:00100000:2.0:1790772435.113163:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772435.113190:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534450560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772435.113212:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772435.113219:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772435.113223:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772435.113233:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1431 00000100:00080000:2.0:1790772435.113277:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772435.113287:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:3.0:1790772435.113289:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790772435.113296:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772435.113317:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534450688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772435.113335:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772435.113341:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772435.113344:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772435.113352:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1431 00000100:00100000:3.0:1790772435.113358:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772435/real 1790772435] req@ffff9c4efbcc1c00 x1877759534450432/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772435.113359:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772435.113368:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772435.113378:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772435.113390:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534450432/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772435.113394:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534450816/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772435.113420:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772435.113422:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772435.113428:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772435.113431:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772435.113431:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772435.113439:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1431 00000100:00100000:3.0:1790772435.113440:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534450432:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772435.113445:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772435.113453:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772435.113461:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772435.113476:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534450944/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772435.113481:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534450560:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772435.113506:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772435.113513:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772435.113517:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1790772435.113521:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772435.113542:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1431 00000100:00080000:2.0:1790772435.113548:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772435.113558:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772435.113561:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772435/real 1790772435] req@ffff9c4efbcc2680 x1877759534450560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790772435.113568:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772435.113591:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534451072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772435.113591:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534450560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772435.113613:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772435.113619:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772435.113622:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772435.113626:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772435.113630:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1431 00000100:00080000:3.0:1790772435.113634:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772435.113637:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772435.113642:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534450560:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772435.113647:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772435.113654:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534450688:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790772435.113656:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772435.113745:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534451200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772435.113746:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772435.113775:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772435.113778:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772435/real 1790772435] req@ffff9c4efbcc2300 x1877759534450688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772435.113783:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772435.113787:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772435.113796:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1431 00000100:00100000:3.0:1790772435.113819:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534450688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772435.113822:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772435.113833:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772435.113838:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790772435.113843:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:3.0:1790772435.113845:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772435.113853:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534450688:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772435.113863:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534451328/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772435.113866:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534450816:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772435.113883:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772435.113890:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772435.113894:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772435.113903:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1431 00000800:00000100:3.0:1790772435.113906:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772435.113910:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772435.113919:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772435.113926:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772435/real 1790772435] req@ffff9c4efbcc1880 x1877759534450816/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790772435.113928:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772435.113949:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534451456/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772435.113951:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534450816/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772435.113973:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772435.113980:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772435.113987:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534450816:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772435.114026:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534450944:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772435.114116:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772435.114140:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772435/real 1790772435] req@ffff9c4efbcc1f80 x1877759534450944/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772435.114181:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534450944/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772435.114201:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772435.114206:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772435.114211:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534450944:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772435.114220:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534451072:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772435.114267:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772435.114281:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772435/real 1790772435] req@ffff9c4efbcc0700 x1877759534451072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772435.114297:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534451072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772435.114317:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772435.114321:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772435.114325:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534451072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772435.114332:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534451200:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772435.114351:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772435.114362:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772435/real 1790772435] req@ffff9c4efbcc0a80 x1877759534451200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772435.114376:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534451200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772435.114387:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772435.114390:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772435.114395:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534451200:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772435.114401:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534451328:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772435.114419:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772435.114430:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772435/real 1790772435] req@ffff9c4efbcc3100 x1877759534451328/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772435.114442:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534451328/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772435.114463:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772435.114467:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772435.114472:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534451328:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772435.114478:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534451456:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772435.114496:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772435.114506:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772435/real 1790772435] req@ffff9c4efbcc0380 x1877759534451456/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772435.114539:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534451456/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772435.114561:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772435.114565:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772435.114570:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534451456:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772435.176741:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772435.176761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772435.176761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772435.240728:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772435.240748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772435.240845:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772435.368752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772435.368794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772435.496773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772435.560774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772435.560798:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772435.560806:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772435.624761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772435.688795:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772435.688802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772435.816779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772435.816814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772435.816818:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772435.880789:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772435.880846:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772435.880851:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772435.880892:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772435.880917:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772436.009788:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772436.072758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772436.136789:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772436.200774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772436.200792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772436.200833:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772436.264785:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772436.264833:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772436.392783:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772436.392851:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772436.520818:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772436.584843:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772436.584844:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772436.584844:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772436.648824:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772436.712792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772436.712795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772436.840824:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772436.840830:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772436.840853:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772436.904862:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772436.904864:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772436.904864:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772436.904946:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772437.032870:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772437.096791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772437.160741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772437.224828:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772437.224849:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772437.224874:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772437.288791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772437.288905:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772437.416759:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772437.416787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772437.544815:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772437.608770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772437.608816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772437.608818:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772437.672761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772437.736790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772437.736851:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772437.864784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772437.864793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772437.864803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772437.928779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772437.928799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772437.928820:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772437.928870:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772437.928881:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772438.056779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772438.120740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772438.184744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772438.248745:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772438.248745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772438.248786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772438.312765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772438.312766:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772438.440713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772438.440732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772438.568720:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772438.632765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772438.632765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772438.632772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772438.696736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772438.760760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772438.760772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772438.888743:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772438.888750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772438.888780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772438.952747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772438.952747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772438.952785:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772438.952788:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772438.952819:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772439.080810:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772439.144751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772439.208750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772439.272773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772439.272794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772439.272834:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772439.336784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772439.336797:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772439.464770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772439.464799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772439.592889:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772439.656810:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772439.656840:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772439.656844:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772439.720769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772439.784834:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772439.784835:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772439.912794:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772439.912897:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772439.912897:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772439.976815:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772439.976891:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772439.976892:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772439.976923:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772439.976954:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772440.104748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772440.168763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772440.232815:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772440.233202:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534451584:192.168.204.150@tcp:250:kworker.0 00000800:00000100:3.0:1790772440.233370:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772440.233447:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772440/real 1790772440] req@ffff9c4efac4df80 x1877759534451584/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772440.233491:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534451584/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772440.233548:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772440.233572:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772440.233587:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534451584:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772440.233654:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534451712:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772440.233770:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772440.233800:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772440/real 1790772440] req@ffff9c4efac4dc00 x1877759534451712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772440.233832:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534451712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772440.233891:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772440.233905:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772440.233917:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534451712:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772440.233936:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534451840:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772440.234038:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772440.234089:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772440/real 1790772440] req@ffff9c4efac4f800 x1877759534451840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772440.234157:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534451840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772440.234201:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772440.234208:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772440.234213:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534451840:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772440.234243:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534451968:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772440.234270:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772440.234281:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772440/real 1790772440] req@ffff9c4efac4f480 x1877759534451968/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772440.234316:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534451968/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772440.234330:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772440.234334:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772440.234338:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534451968:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772440.234346:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534452096:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772440.234369:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772440.234381:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772440/real 1790772440] req@ffff9c4efac4e300 x1877759534452096/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772440.234396:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534452096/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772440.234417:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772440.234422:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772440.234426:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534452096:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772440.234449:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534452224:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772440.234470:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772440.234482:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772440/real 1790772440] req@ffff9c4efac4c700 x1877759534452224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772440.234497:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534452224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772440.234528:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772440.234533:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772440.234538:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534452224:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772440.234545:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534452352:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772440.234564:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772440.234576:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772440/real 1790772440] req@ffff9c4efac4ed80 x1877759534452352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772440.234591:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534452352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772440.234601:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772440.234605:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772440.234610:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534452352:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772440.234618:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534452480:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772440.234656:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772440.234673:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772440/real 1790772440] req@ffff9c4efac4ea00 x1877759534452480/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772440.234686:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534452480/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772440.234770:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772440.234776:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772440.234781:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534452480:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772440.234789:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534452608:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772440.234819:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772440.234838:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772440/real 1790772440] req@ffff9c4efac4e680 x1877759534452608/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772440.234854:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534452608/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772440.234895:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772440.234899:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772440.234903:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534452608:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772440.296744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772440.296778:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772440.296784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772440.360789:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772440.360884:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772440.488755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772440.488757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772440.616765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772440.680787:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772440.680795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772440.680805:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772440.744768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772440.808751:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772440.808813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772440.936754:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772440.936824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772440.936835:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772441.000779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772441.000779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772441.000788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772441.000841:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772441.128749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772441.192713:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772441.256783:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772441.320771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772441.320790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772441.320856:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772441.384872:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772441.384874:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772441.512810:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772441.512832:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772441.640797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772441.704758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772441.704823:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772441.704861:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772441.768758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772441.832787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772441.832981:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772441.960801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772441.960839:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772441.960853:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772442.024783:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772442.024790:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772442.024805:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772442.024870:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772442.024875:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772442.152783:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772442.216770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772442.280804:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772442.344846:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772442.344847:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772442.344905:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772442.408771:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772442.408776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772442.536853:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772442.536857:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772442.664767:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772442.728807:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772442.728814:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772442.728877:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772442.792743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772442.856796:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772442.856811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772442.984771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772442.984781:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772442.984841:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772443.048839:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772443.048926:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772443.048937:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772443.048941:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772443.176716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772443.240814:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772443.304755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772443.368787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772443.368789:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772443.368800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772443.432768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772443.432835:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772443.560758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772443.560793:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772443.688794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772443.752763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772443.752773:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772443.752787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772443.816741:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772443.880792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772443.880819:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772444.008794:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772444.008796:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772444.008845:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772444.072794:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772444.072804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772444.072871:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772444.072920:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772444.200771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772444.264734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772444.328724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772444.392817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772444.392817:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772444.392824:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772444.456774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772444.456938:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772444.584823:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772444.584824:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772444.712745:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772444.776757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772444.776799:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772444.776804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772444.840739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772444.904770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772444.904825:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772445.032776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772445.032799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772445.032807:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772445.096787:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772445.096858:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772445.096939:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772445.096972:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772445.097009:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772445.224756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772445.288800:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772445.352723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772445.353059:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534452736:192.168.204.150@tcp:250:kworker.0 00000800:00000100:3.0:1790772445.353198:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772445.353300:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772445/real 1790772445] req@ffff9c4efac4fb80 x1877759534452736/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772445.353417:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534452736/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772445.353494:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772445.353513:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772445.353544:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534452736:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772445.353621:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534452864:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772445.353728:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772445.353760:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772445/real 1790772445] req@ffff9c4efac4e680 x1877759534452864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772445.353802:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534452864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772445.353839:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772445.353847:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772445.353853:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534452864:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772445.353889:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534452992:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772445.353926:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772445.353953:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772445/real 1790772445] req@ffff9c4efac4ea00 x1877759534452992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772445.353969:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534452992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772445.353981:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772445.353985:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772445.353990:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534452992:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772445.353997:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534453120:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772445.354017:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772445.354028:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772445/real 1790772445] req@ffff9c4efac4ed80 x1877759534453120/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772445.354041:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534453120/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772445.354052:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772445.354056:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772445.354059:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534453120:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772445.354066:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534453248:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772445.354083:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772445.354094:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772445/real 1790772445] req@ffff9c4efac4c700 x1877759534453248/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772445.354107:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534453248/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772445.354123:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772445.354129:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772445.354135:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534453248:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772445.354146:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534453376:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772445.354180:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772445.354200:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772445/real 1790772445] req@ffff9c4efac4e300 x1877759534453376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772445.354214:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534453376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772445.354235:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772445.354239:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772445.354243:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534453376:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772445.354251:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534453504:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772445.354273:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772445.354283:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772445/real 1790772445] req@ffff9c4efac4f480 x1877759534453504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772445.354295:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534453504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772445.354306:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772445.354310:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772445.354314:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534453504:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772445.354342:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534453632:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772445.354374:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772445.354393:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772445/real 1790772445] req@ffff9c4efac4f800 x1877759534453632/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772445.354415:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534453632/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772445.354431:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772445.354436:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772445.354440:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534453632:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772445.354447:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534453760:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772445.354472:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772445.354484:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772445/real 1790772445] req@ffff9c4efac4dc00 x1877759534453760/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772445.354497:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534453760/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772445.354509:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772445.354514:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772445.354521:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534453760:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772445.416767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772445.416888:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772445.416899:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772445.480770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772445.480821:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772445.608751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772445.608803:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772445.736791:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772445.800780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772445.800782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772445.800811:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772445.864767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772445.928777:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772445.928804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772446.056815:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772446.056816:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772446.056834:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772446.120791:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772446.120807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772446.120876:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772446.120918:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772446.120936:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772446.248719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772446.312750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772446.376777:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772446.440801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772446.440805:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772446.440806:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772446.504900:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772446.504963:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772446.504999:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772446.632767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772446.632813:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772446.760953:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772446.824758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772446.824775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772446.824790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772446.888743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772446.952800:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772446.952814:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772447.080771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772447.080817:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772447.080820:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772447.144851:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772447.144851:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772447.144871:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772447.144930:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772447.272759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772447.336773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772447.400764:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772447.464794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772447.464807:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772447.464814:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772447.528785:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772447.528804:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772447.529020:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772447.656758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772447.656790:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772447.784783:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772447.848793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772447.848793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772447.848852:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772447.912789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772447.976773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772447.976868:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772448.104849:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772448.104862:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772448.104869:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772448.168874:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772448.168884:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772448.168932:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772448.168967:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772448.168987:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772448.296803:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772448.360841:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772448.424776:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772448.488826:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772448.488844:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772448.488845:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772448.552834:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772448.552899:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772448.552936:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772448.680739:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772448.680789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772448.808880:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772448.872746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772448.872819:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772448.872828:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772448.936726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772449.000749:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772449.000770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772449.128780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772449.128820:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772449.128821:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772449.192777:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772449.192836:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772449.192871:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772449.192905:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772449.192917:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772449.320757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772449.384757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772449.448773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772449.512778:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772449.512827:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772449.512841:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772449.576794:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772449.576802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772449.576913:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772449.704759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772449.704817:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772449.832765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772449.896850:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772449.896850:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772449.896850:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772449.960825:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772450.024796:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772450.024832:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772450.152799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772450.152815:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772450.152829:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772450.216807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772450.216818:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772450.216822:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772450.216873:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772450.216880:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772450.344749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772450.408873:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772450.472807:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772450.473116:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534453888:192.168.204.150@tcp:250:kworker.0 00000800:00000100:3.0:1790772450.473243:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772450.473321:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772450/real 1790772450] req@ffff9c4efac4df80 x1877759534453888/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772450.473385:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534453888/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772450.473427:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772450.473444:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772450.473459:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534453888:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772450.473507:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534454016:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772450.473568:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772450.473609:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772450/real 1790772450] req@ffff9c4efac4dc00 x1877759534454016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772450.473694:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534454016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772450.473724:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772450.473740:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772450.473753:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534454016:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772450.473804:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534454144:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772450.473851:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772450.473874:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772450/real 1790772450] req@ffff9c4efac4f800 x1877759534454144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772450.473905:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534454144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772450.473961:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772450.473977:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772450.473994:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534454144:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772450.474026:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534454272:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772450.474090:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772450.474113:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772450/real 1790772450] req@ffff9c4efac4f480 x1877759534454272/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772450.474129:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534454272/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772450.474152:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772450.474157:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772450.474162:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534454272:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772450.474171:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534454400:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772450.474191:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772450.474203:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772450/real 1790772450] req@ffff9c4efac4e300 x1877759534454400/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772450.474217:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534454400/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772450.474230:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772450.474234:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772450.474238:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534454400:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772450.474262:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534454528:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772450.474283:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772450.474296:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772450/real 1790772450] req@ffff9c4efac4c700 x1877759534454528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772450.474311:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534454528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772450.474346:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772450.474354:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772450.474362:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534454528:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772450.474375:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534454656:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772450.474414:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772450.474438:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772450/real 1790772450] req@ffff9c4efac4ed80 x1877759534454656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772450.474458:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534454656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772450.474470:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772450.474474:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772450.474479:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534454656:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772450.474487:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534454784:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772450.474514:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772450.474527:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772450/real 1790772450] req@ffff9c4efac4ea00 x1877759534454784/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772450.474542:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534454784/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772450.474555:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772450.474559:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772450.474564:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534454784:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772450.474571:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534454912:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772450.474591:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772450.474611:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772450/real 1790772450] req@ffff9c4efac4e680 x1877759534454912/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772450.474626:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534454912/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772450.474639:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772450.474643:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772450.474647:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534454912:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772450.536813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772450.536822:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772450.536836:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772450.600751:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772450.600862:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772450.600943:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772450.728712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772450.728744:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772450.856715:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772450.920715:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772450.920715:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772450.920721:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772450.984748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772451.048795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772451.048804:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772451.176789:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772451.176805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772451.176811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772451.240824:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772451.240827:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772451.240850:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772451.240879:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772451.240900:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772451.368770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772451.432767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772451.496812:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772451.560799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772451.560821:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772451.560825:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772451.624825:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772451.624901:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772451.624986:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772451.752793:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772451.752823:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772451.880810:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772451.944761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772451.944761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772451.944762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772452.008777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772452.072812:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772452.072821:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772452.200791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772452.200814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772452.200832:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772452.264789:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772452.264833:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772452.264841:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772452.264882:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772452.392771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772452.456754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772452.520793:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772452.584763:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772452.584819:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772452.584823:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772452.648805:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772452.648851:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772452.649043:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772452.776773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772452.776790:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772452.904798:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772452.968796:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772452.968822:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772452.968828:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772453.032789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772453.096778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772453.096812:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772453.224798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772453.224819:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772453.224828:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772453.288799:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772453.288815:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772453.288847:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772453.288885:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772453.288912:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772453.416777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772453.480803:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772453.544824:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772453.608812:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772453.608818:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772453.608834:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772453.672781:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772453.672802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772453.672944:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772453.800752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772453.800755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772453.928834:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772453.992767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772453.992767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772453.992767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772454.056735:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772454.120742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772454.120742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772454.248797:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772454.248797:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772454.248798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772454.312760:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772454.312766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772454.312783:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772454.312801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772454.440744:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772454.504754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772454.568797:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772454.632807:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772454.632831:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772454.632853:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772454.696850:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772454.696881:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772454.696908:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772454.824805:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772454.824816:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772454.952828:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772455.016792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772455.016825:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772455.016829:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772455.080819:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772455.144798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772455.144806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772455.272782:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772455.272794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772455.272810:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772455.336797:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772455.336807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772455.336884:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772455.336887:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772455.336900:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772455.464815:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772455.528779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772455.592780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772455.592841:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772455.592856:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772455.592866:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772455.592887:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1451 00000100:00080000:0.0:1790772455.592910:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772455.592935:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772455.593004:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534455040/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772455.593048:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772455.593053:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772455.593057:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772455.593063:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1451 00000100:00080000:0.0:1790772455.593069:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772455.593080:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772455.593097:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772455.593110:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534455040:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772455.593121:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534455168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772455.593140:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772455.593145:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772455.593149:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772455.593155:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1451 00000100:00080000:0.0:1790772455.593160:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772455.593168:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772455.593175:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772455.593193:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534455296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772455.593208:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772455.593214:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772455.593217:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772455.593224:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1451 00000100:00080000:0.0:1790772455.593231:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772455.593237:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772455.593245:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772455.593260:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534455424/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772455.593275:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772455.593281:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772455.593285:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772455.593293:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1451 00000100:00080000:0.0:1790772455.593299:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772455.593308:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772455.593317:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534455168:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790772455.593318:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772455.593345:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534455552/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772455.593364:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772455.593369:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772455.593373:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772455.593380:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1451 00000100:00100000:3.0:1790772455.593388:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534455296:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772455.593389:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772455.593405:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772455.593414:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772455.593435:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534455680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772455.593441:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534455424:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790772455.593467:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772455.593473:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772455.593476:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772455.593486:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1451 00000100:00080000:0.0:1790772455.593494:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772455.593502:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772455.593511:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772455.593523:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534455552:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772455.593532:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534455808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772455.593553:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772455.593558:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772455.593562:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772455.593569:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1451 00000100:00080000:0.0:1790772455.593577:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772455.593586:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772455.593594:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772455.593608:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534455680:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772455.593622:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534455936/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772455.593721:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772455.593732:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772455.593738:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772455.593752:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1451 00000100:00100000:3.0:1790772455.593754:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534455808:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772455.593768:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772455.593786:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772455.593801:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772455.593802:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534455936:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772455.593846:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534456064/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772455.593903:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534456064:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772455.656801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772455.656802:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772455.656835:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772455.720836:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772455.720889:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772455.720936:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772455.848771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772455.848787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772455.976801:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772456.040779:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772456.040795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772456.040795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772456.104804:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772456.168781:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772456.168788:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772456.296765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772456.296773:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772456.296788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772456.360779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772456.360798:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772456.360808:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772456.488760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772456.552764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772456.616730:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772456.680745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772456.680764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772456.680766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772456.744811:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772456.744816:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772456.744884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772456.872751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772456.872751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772457.000786:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772457.064782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772457.064835:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772457.064853:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772457.128745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772457.192787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772457.192797:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772457.320804:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772457.320809:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772457.320809:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772457.384818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772457.384844:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772457.384844:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772457.384911:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772457.512759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772457.576750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772457.640761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772457.704788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772457.704789:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772457.704795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772457.768728:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772457.768737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772457.768869:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772457.896788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772457.896788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772458.024765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772458.088804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772458.088808:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772458.088809:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772458.216766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772458.216805:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772458.344791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772458.344812:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772458.344814:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772458.408778:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772458.408817:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772458.408870:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772458.408877:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772458.408897:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772458.536760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772458.600759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772458.664721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:2.0:1790772458.665063:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:2.0:1790772458.665128:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1790772458.665155:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790772458.665230:0:2383:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:2.0:1790772458.665252:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790772458.665269:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790772458.665282:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790772458.665296:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790772458.665320:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790772458.665339:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:3.0:1790772458.665345:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772455/real 1790772458] req@ffff9c4ec7a7d180 x1877759534456064/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1790772458.665359:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:3.0:1790772458.665404:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534456064/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772458.665428:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772458.665436:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772458.665443:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534456064:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772458.665454:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772455/real 1790772458] req@ffff9c4ec7a7c700 x1877759534455808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772458.665482:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534455808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772458.665500:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772458.665508:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772458.665516:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534455808:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772458.665530:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772455/real 1790772458] req@ffff9c4ec7a7fb80 x1877759534455936/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772458.665554:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534455936/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772458.665576:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772458.665583:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772458.665590:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534455936:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772458.665644:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772455/real 1790772458] req@ffff9c4ec7a7e300 x1877759534455552/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772458.665720:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534455552/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772458.665753:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772458.665761:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772458.665770:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534455552:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772458.665784:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772455/real 1790772458] req@ffff9c4ec7a7ca80 x1877759534455680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772458.665806:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534455680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772458.665834:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772458.665842:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772458.665850:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534455680:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772458.665864:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772455/real 1790772458] req@ffff9c4ec7a7ed80 x1877759534455168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772458.665890:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534455168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772458.665921:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772458.665932:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772458.665943:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534455168:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772458.665961:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772455/real 1790772458] req@ffff9c4ec7a7d880 x1877759534455296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772458.665987:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534455296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772458.666002:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772458.666009:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772458.666015:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534455296:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772458.666028:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772455/real 1790772458] req@ffff9c4ec7a7c380 x1877759534455424/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772458.666050:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534455424/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772458.666068:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772458.666074:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772458.666080:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534455424:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772458.666092:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772455/real 1790772458] req@ffff9c4ec7a7f480 x1877759534455040/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772458.666116:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534455040/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772458.666537:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772458.666548:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772458.666556:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534455040:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772458.728767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772458.728772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772458.728817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772458.792712:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772458.792720:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772458.792826:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772458.920725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772458.920728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772459.048719:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772459.112718:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772459.112719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772459.112719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772459.240717:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772459.240720:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772459.368750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772459.368772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772459.368783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772459.432769:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772459.432791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772459.432986:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772459.560717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772459.624725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772459.688789:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772459.752786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772459.752804:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772459.816761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772459.816761:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772459.817024:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772459.944789:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772459.944791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772460.072779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772460.136761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772460.136767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772460.136790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772460.264755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772460.264790:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772460.392776:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772460.392784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772460.392803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772460.456794:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772460.456811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772460.456822:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772460.456881:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772460.456908:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772460.584771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772460.648758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772460.712762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772460.712786:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772460.712797:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772460.712804:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772460.712822:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1456 00000100:00080000:0.0:1790772460.712839:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772460.712860:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772460.712922:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534456192/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772460.712966:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772460.712971:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772460.712974:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772460.712980:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1456 00000100:00080000:0.0:1790772460.712985:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772460.712993:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772460.713013:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772460.713027:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534456192:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772460.713037:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534456320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772460.713071:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772460.713076:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772460.713079:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772460.713086:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1456 00000100:00080000:0.0:1790772460.713092:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772460.713098:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772460.713105:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772460.713122:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534456448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772460.713136:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772460.713141:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772460.713144:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772460.713151:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1456 00000100:00080000:0.0:1790772460.713156:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772460.713162:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772460.713170:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772460.713179:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772460.713192:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534456576/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772460.713222:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772460.713227:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772460.713231:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772460.713238:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1456 00000100:00080000:0.0:1790772460.713244:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772460.713256:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772460.713268:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772460.713272:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772460/real 1790772460] req@ffff9c4ec7a7df80 x1877759534456192/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772460.713299:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534456704/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772460.713323:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772460.713325:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534456192/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772460.713338:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772460.713348:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772460.713368:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1456 00000100:00080000:3.0:1790772460.713370:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772460.713381:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772460.713400:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772460.713411:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1790772460.713416:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772460.713434:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534456192:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772460.713441:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534456832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772460.713468:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772460.713475:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772460.713479:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772460.713494:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1456 00000100:00100000:3.0:1790772460.713494:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534456320:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772460.713518:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772460.713530:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772460.713542:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772460.713597:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534456960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772460.713605:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772460.713633:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772460.713642:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1790772460.713647:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772460/real 1790772460] req@ffff9c4ec7a7f480 x1877759534456320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772460.713725:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534456320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772460.713732:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772460.713757:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1456 00000100:00080000:0.0:1790772460.713771:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772460.713777:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772460.713788:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772460.713795:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790772460.713808:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772460.713809:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534456320:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772460.713853:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534456448:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772460.713855:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534457088/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772460.713884:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772460.713892:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772460.713896:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772460.713909:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1456 00000100:00080000:0.0:1790772460.713919:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772460.713940:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772460.713952:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772460.713976:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534457216/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772460.714582:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772460.714628:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772460/real 1790772460] req@ffff9c4ec7a7c380 x1877759534456448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772460.714835:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534456448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772460.714912:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772460.714921:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772460.714929:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534456448:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772460.714944:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534456576:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772460.715002:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772460.715029:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772460/real 1790772460] req@ffff9c4ec7a7d880 x1877759534456576/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772460.715070:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534456576/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772460.715122:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772460.715131:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772460.715139:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534456576:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772460.715170:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534456704:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772460.715226:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772460.715252:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772460/real 1790772460] req@ffff9c4ec7a7ed80 x1877759534456704/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772460.715277:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534456704/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772460.715299:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772460.715308:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772460.715318:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534456704:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772460.715335:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534456832:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772460.715382:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772460.715407:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772460/real 1790772460] req@ffff9c4ec7a7ca80 x1877759534456832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772460.715434:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534456832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772460.715485:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772460.715496:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772460.715503:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534456832:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772460.715546:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534456960:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772460.715614:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772460.715640:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772460/real 1790772460] req@ffff9c4ec7a7e300 x1877759534456960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772460.715711:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534456960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772460.715729:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772460.715734:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772460.715739:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534456960:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772460.715748:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534457088:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772460.715780:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772460.715795:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772460/real 1790772460] req@ffff9c4ec7a7fb80 x1877759534457088/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772460.715811:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534457088/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772460.715824:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772460.715828:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772460.715834:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534457088:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772460.715842:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534457216:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772460.715863:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772460.715875:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772460/real 1790772460] req@ffff9c4ec7a7c700 x1877759534457216/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772460.715890:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534457216/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772460.715903:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772460.715907:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772460.715911:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534457216:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772460.776753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772460.776787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772460.776875:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772460.840767:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772460.840812:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772460.840976:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772460.968763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772460.968781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772461.096875:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772461.160754:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772461.160760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772461.160790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772461.288778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772461.288788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772461.416766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772461.416805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772461.416805:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772461.480807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772461.480810:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772461.480825:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772461.480855:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772461.608767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772461.672737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772461.736786:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772461.800764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772461.800786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772461.800796:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772461.864751:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772461.864784:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772461.864936:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772461.992805:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772461.992828:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772462.120791:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772462.184781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772462.184788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772462.184816:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772462.248865:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772462.312787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772462.312790:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772462.440805:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772462.440806:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772462.440806:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772462.504776:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772462.504776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772462.504830:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772462.504904:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772462.504918:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772462.632753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772462.696764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772462.760787:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772462.824786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772462.824788:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772462.824873:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772462.888831:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772462.888888:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772462.888995:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772463.016753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772463.016768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772463.144800:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772463.208793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772463.208802:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772463.208805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772463.272752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772463.336751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772463.336847:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772463.464761:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772463.464845:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772463.464868:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772463.528821:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772463.528840:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772463.528863:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772463.528874:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772463.528903:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772463.656738:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772463.720768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772463.784772:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772463.848763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772463.848780:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772463.848813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772463.912766:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772463.912832:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772463.912880:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772464.040764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772464.040787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772464.168814:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772464.232774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772464.232793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772464.232818:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772464.296786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772464.360782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772464.360825:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772464.488747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772464.488780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772464.488801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772464.552782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772464.552822:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772464.552849:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772464.552879:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772464.680761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772464.744793:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772464.808739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772464.872800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772464.872864:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772464.872878:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772464.936780:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772464.936840:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772464.936940:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772465.064745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772465.065122:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772465.192786:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772465.256762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772465.256776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772465.256803:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772465.320753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772465.384775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772465.384881:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772465.512782:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772465.512802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772465.512811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772465.576776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772465.576828:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772465.576839:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772465.576942:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772465.704773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772465.768757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772465.832747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772465.833050:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534457344:192.168.204.150@tcp:250:kworker.0 00000800:00000100:3.0:1790772465.833175:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772465.833252:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772465/real 1790772465] req@ffff9c4efac4fb80 x1877759534457344/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772465.833257:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772465.833288:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534457856/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772465.833300:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534457344/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772465.833311:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772465.833327:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772465.833334:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772465.833342:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772465.833344:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1461 00000100:00080000:1.0:1790772465.833356:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772465.833360:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772465.833368:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772465.833372:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534457344:192.168.204.150@tcp:250:kworker.0 00000008:00100000:1.0:1790772465.833379:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772465.833398:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534457984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772465.833411:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772465.833419:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1790772465.833420:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534457472:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772465.833424:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772465.833436:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1461 00000100:00080000:1.0:1790772465.833441:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772465.833450:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772465.833458:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1790772465.833491:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772465.833510:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534458112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772465.833521:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772465/real 1790772465] req@ffff9c4efac4e680 x1877759534457472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772465.833536:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772465.833546:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772465.833549:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772465.833553:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534457472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772465.833563:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1461 00000100:00080000:1.0:1790772465.833575:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772465.833588:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772465.833597:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:3.0:1790772465.833601:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772465.833611:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790772465.833618:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534458240/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772465.833626:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534457472:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772465.833681:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534457600:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772465.833700:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772465.833706:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772465.833709:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772465.833721:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1461 00000800:00000100:3.0:1790772465.833722:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772465.833736:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772465.833748:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772465.833753:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772465/real 1790772465] req@ffff9c4efac4ea00 x1877759534457600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772465.833759:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772465.833785:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534457600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772465.833792:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534458368/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772465.833854:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772465.833861:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772465.833868:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534457600:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772465.833878:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534457728:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772465.833904:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772465.833917:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772465/real 1790772465] req@ffff9c4efac4ed80 x1877759534457728/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772465.833932:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534457728/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772465.833946:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772465.833950:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772465.833954:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534457728:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772465.833962:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534457856:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772465.833992:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772465.834005:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772465/real 1790772465] req@ffff9c4efac4c700 x1877759534457856/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772465.834021:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534457856/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772465.834034:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772465.834038:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772465.834044:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534457856:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772465.834052:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534457984:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772465.834073:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772465.834085:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772465/real 1790772465] req@ffff9c4efac4e300 x1877759534457984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772465.834100:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534457984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772465.834120:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772465.834124:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772465.834129:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534457984:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772465.834153:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534458112:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772465.834174:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772465.834187:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772465/real 1790772465] req@ffff9c4efac4f480 x1877759534458112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772465.834202:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534458112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772465.834213:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772465.834216:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772465.834221:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534458112:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772465.834228:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534458240:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772465.834247:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772465.834259:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772465/real 1790772465] req@ffff9c4efac4f800 x1877759534458240/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772465.834275:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534458240/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772465.834288:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772465.834292:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772465.834296:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534458240:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772465.834303:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534458368:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772465.834322:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772465.834334:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772465/real 1790772465] req@ffff9c4efac4dc00 x1877759534458368/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772465.834348:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534458368/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772465.834361:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772465.834365:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772465.834369:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534458368:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772465.896774:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772465.896780:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772465.896814:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772465.896822:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772465.960786:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772465.960843:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772465.960998:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772466.088769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772466.088782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772466.152793:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772466.216803:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772466.280777:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772466.280789:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772466.280793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772466.344765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772466.408785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772466.408798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772466.408800:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772466.536879:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772466.536886:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772466.536892:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772466.536916:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772466.600751:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772466.600782:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772466.600790:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772466.600801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772466.600824:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772466.600826:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772466.728761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772466.792765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772466.856760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772466.920758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772466.920771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772466.920800:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772466.920804:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772466.984780:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772466.984834:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772466.985013:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772467.112823:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772467.112876:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772467.176752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772467.240734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772467.304769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772467.304806:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772467.304809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772467.368766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772467.432808:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772467.432828:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772467.432831:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772467.560779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772467.560789:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772467.560815:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772467.560817:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772467.624747:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772467.624804:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772467.624805:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772467.624818:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772467.624836:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772467.624873:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772467.752766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772467.816782:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772467.880794:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772467.944865:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772467.944873:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772467.944902:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772467.944902:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772468.008783:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772468.008829:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772468.008976:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772468.136743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772468.136786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772468.200769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772468.264892:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772468.328753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772468.328790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772468.328790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772468.392756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772468.456921:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772468.456928:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772468.456955:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772468.584779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772468.584821:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772468.584825:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772468.584828:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772468.648768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772468.648805:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772468.648818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772468.648826:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772468.648850:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772468.648898:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772468.776780:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772468.840775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772468.904742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772468.968768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772468.968778:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772468.968815:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772468.968815:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772469.032793:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772469.032798:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772469.032970:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772469.160752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772469.160761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772469.224761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772469.288745:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772469.352781:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772469.352791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772469.352796:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772469.416809:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772469.480772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772469.480773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772469.480793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772469.608770:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772469.608775:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772469.608798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772469.608798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772469.672780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772469.672802:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772469.672817:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772469.672831:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772469.672866:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772469.672873:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772469.800721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772469.864754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772469.928743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772469.992766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772469.992780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772469.992783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772469.992790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772470.056740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772470.056803:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772470.056948:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772470.184768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772470.184768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772470.248762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772470.312741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772470.376764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772470.376774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772470.376778:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772470.440756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772470.504777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772470.504786:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772470.504796:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772470.632757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772470.632767:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772470.632783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772470.632796:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772470.696765:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772470.696800:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772470.696806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772470.696826:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772470.696838:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772470.696851:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772470.824754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772470.888744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772470.952760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772470.952765:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772470.952777:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772470.952784:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772470.952800:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1466 00000100:00080000:1.0:1790772470.952817:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772470.952836:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772470.952905:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534458496/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772470.952958:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772470.952964:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772470.952969:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772470.952978:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1466 00000100:00080000:1.0:1790772470.952983:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772470.952991:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772470.953008:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772470.953018:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534458496:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772470.953036:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534458624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772470.953054:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772470.953060:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772470.953062:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772470.953069:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1466 00000100:00080000:1.0:1790772470.953076:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772470.953082:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772470.953089:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772470.953106:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534458752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772470.953120:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772470.953125:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772470.953127:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772470.953134:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1466 00000800:00000100:3.0:1790772470.953143:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772470.953144:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772470.953157:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772470.953165:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772470.953183:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534458880/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772470.953196:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772470.953201:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772470.953203:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772470.953208:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1466 00000100:00080000:1.0:1790772470.953214:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772470.953220:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772470.953228:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772470.953236:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772470/real 1790772470] req@ffff9c4efac4df80 x1877759534458496/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772470.953250:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534459008/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772470.953273:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772470.953280:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1790772470.953283:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534458496/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772470.953284:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772470.953301:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1466 00000100:00080000:1.0:1790772470.953310:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772470.953322:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772470.953324:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790772470.953329:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1790772470.953337:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772470.953347:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534458496:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772470.953353:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534459136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772470.953395:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772470.953400:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772470.953402:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772470.953411:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1466 00000100:00080000:1.0:1790772470.953418:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772470.953419:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534458624:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772470.953431:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772470.953441:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772470.953461:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534459264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772470.953475:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772470.953492:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772470.953498:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772470.953501:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772470.953508:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772470/real 1790772470] req@ffff9c4efac4dc00 x1877759534458624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772470.953509:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1466 00000100:00080000:1.0:1790772470.953533:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772470.953542:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534458624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772470.953548:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772470.953562:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790772470.953585:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534459392/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772470.953586:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772470.953601:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772470.953615:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534458624:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772470.953621:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772470.953628:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1790772470.953633:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534458752:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772470.953634:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772470.953687:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1466 00000100:00080000:1.0:1790772470.953693:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772470.953701:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772470.953710:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772470.953729:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534459520/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772470.953737:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772470.953764:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772470/real 1790772470] req@ffff9c4efac4f800 x1877759534458752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772470.953786:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534458752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772470.953798:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772470.953803:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772470.953807:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534458752:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772470.953816:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534458880:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772470.953840:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772470.953852:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772470/real 1790772470] req@ffff9c4efac4f480 x1877759534458880/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772470.953867:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534458880/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772470.953881:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772470.953885:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772470.953889:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534458880:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772470.953897:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534459008:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772470.953917:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772470.953929:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772470/real 1790772470] req@ffff9c4efac4e300 x1877759534459008/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772470.953954:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534459008/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772470.953967:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772470.953971:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772470.953975:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534459008:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772470.953983:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534459136:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772470.954002:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772470.954015:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772470/real 1790772470] req@ffff9c4efac4c700 x1877759534459136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772470.954030:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534459136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772470.954050:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772470.954054:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772470.954058:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534459136:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772470.954083:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534459264:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772470.954105:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772470.954118:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772470/real 1790772470] req@ffff9c4efac4ed80 x1877759534459264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772470.954133:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534459264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772470.954153:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772470.954157:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772470.954162:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534459264:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772470.954169:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534459392:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772470.954187:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772470.954198:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772470/real 1790772470] req@ffff9c4efac4ea00 x1877759534459392/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772470.954213:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534459392/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772470.954227:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772470.954231:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772470.954235:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534459392:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772470.954243:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534459520:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772470.954262:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772470.954274:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772470/real 1790772470] req@ffff9c4efac4e680 x1877759534459520/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772470.954289:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534459520/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772470.954301:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772470.954305:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772470.954310:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534459520:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772471.016812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772471.016815:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772471.016815:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772471.016852:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772471.080869:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772471.080871:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772471.080919:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772471.208785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772471.208798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772471.272769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772471.336766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772471.400785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772471.400786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772471.400798:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772471.464769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772471.528881:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772471.528882:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772471.528882:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772471.656775:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772471.656785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772471.656785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772471.656808:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772471.720735:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772471.720751:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772471.720760:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772471.720767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772471.721712:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772471.721737:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772471.848728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772471.912755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772471.976752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772472.040745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772472.040755:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772472.040757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772472.040792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772472.104764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772472.104815:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772472.104928:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772472.232792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772472.232841:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772472.296781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772472.360761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772472.424777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772472.424781:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772472.424782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772472.488750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772472.552739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772472.552742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772472.552747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772472.680750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772472.680760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772472.680765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772472.680781:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772472.744747:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772472.744756:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772472.744784:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772472.744817:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772472.744818:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772472.744851:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772472.872764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772472.936755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772473.001041:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772473.064792:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772473.064792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772473.064792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772473.064793:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772473.128747:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772473.128796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772473.128869:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772473.256747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772473.256807:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772473.320749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772473.384752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772473.448788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772473.448797:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772473.448830:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772473.512745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772473.576761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772473.576785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772473.576790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772473.704770:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772473.704793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772473.704804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772473.704811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772473.768748:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772473.768767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772473.768801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772473.768801:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772473.768822:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772473.768842:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772473.896746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772473.960757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772474.024755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772474.088776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772474.088786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772474.088786:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772474.088789:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772474.152795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772474.152839:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772474.152877:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772474.280786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772474.280801:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772474.344764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772474.408779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772474.472757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772474.472768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772474.472794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772474.536795:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772474.600772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772474.600779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772474.600791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772474.728763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772474.728771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772474.728786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772474.728810:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772474.792763:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772474.792768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772474.792812:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772474.792812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772474.792856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772474.792858:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772474.920726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772474.984766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772475.048781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772475.112789:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772475.112796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772475.112856:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772475.112865:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772475.176797:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772475.176810:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772475.176949:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772475.304771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772475.304795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772475.368771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772475.432912:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772475.496780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772475.496787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772475.496818:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772475.560759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772475.624779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772475.624802:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772475.624805:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772475.752764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772475.752777:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772475.752805:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772475.752840:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772475.816793:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772475.816801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772475.816868:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772475.816869:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772475.816921:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772475.816951:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772475.944773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772476.008732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772476.072745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772476.072782:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772476.072796:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772476.072806:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772476.072825:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1471 00000100:00080000:1.0:1790772476.072856:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772476.072875:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772476.072945:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534459648/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772476.072998:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772476.073004:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772476.073007:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772476.073013:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1471 00000100:00080000:1.0:1790772476.073019:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772476.073027:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772476.073042:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772476.073066:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534459776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772476.073081:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772476.073087:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534459648:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772476.073088:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772476.073092:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772476.073105:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1471 00000100:00080000:1.0:1790772476.073113:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772476.073123:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772476.073133:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772476.073160:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534459904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772476.073182:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772476.073188:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772476.073191:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772476.073199:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1471 00000100:00080000:1.0:1790772476.073205:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772476.073213:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772476.073221:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772476.073223:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772476.073243:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534460032/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772476.073255:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772476.073262:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772476.073289:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772476.073300:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1471 00000100:00080000:1.0:1790772476.073309:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772476.073320:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772476.073331:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772476.073353:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772476/real 1790772476] req@ffff9c4efac4fb80 x1877759534459648/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772476.073361:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534460160/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772476.073406:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772476.073412:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772476.073415:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772476.073424:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1471 00000100:00100000:3.0:1790772476.073427:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534459648/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772476.073431:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772476.073443:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772476.073450:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772476.073467:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534460288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772476.073474:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772476.073481:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772476.073487:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772476.073493:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772476.073493:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772476.073506:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1471 00000100:00100000:3.0:1790772476.073513:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534459648:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772476.073516:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772476.073534:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772476.073544:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772476.073566:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534460416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772476.073572:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534459776:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772476.073604:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772476.073610:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772476.073613:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772476.073621:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1471 00000100:00080000:1.0:1790772476.073628:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772476.073646:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:3.0:1790772476.073648:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772476.073696:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790772476.073726:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534460544/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772476.073741:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772476/real 1790772476] req@ffff9c4efac4e680 x1877759534459776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772476.073759:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772476.073774:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772476.073778:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772476.073788:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1471 00000100:00080000:1.0:1790772476.073797:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772476.073813:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772476.073814:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534459776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772476.073827:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772476.073854:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534460672/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772476.073864:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772476.073877:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772476.073886:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534459776:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772476.073902:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534459904:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772476.073944:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772476.073967:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772476/real 1790772476] req@ffff9c4efac4ea00 x1877759534459904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772476.073990:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534459904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772476.074006:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772476.074013:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772476.074019:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534459904:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772476.074030:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534460032:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772476.074065:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772476.074083:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772476/real 1790772476] req@ffff9c4efac4ed80 x1877759534460032/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772476.074104:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534460032/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772476.074136:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772476.074142:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772476.074149:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534460032:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772476.074160:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534460160:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772476.074189:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772476.074206:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772476/real 1790772476] req@ffff9c4efac4c700 x1877759534460160/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772476.074226:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534460160/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772476.074244:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772476.074251:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772476.074257:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534460160:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772476.074291:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534460288:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772476.074328:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772476.074348:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772476/real 1790772476] req@ffff9c4efac4e300 x1877759534460288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772476.074368:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534460288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772476.074396:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772476.074402:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772476.074410:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534460288:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772476.074442:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534460416:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772476.074478:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772476.074499:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772476/real 1790772476] req@ffff9c4efac4f480 x1877759534460416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772476.074521:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534460416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772476.074537:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772476.074544:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772476.074550:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534460416:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772476.074561:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534460544:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772476.074593:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772476.074611:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772476/real 1790772476] req@ffff9c4efac4f800 x1877759534460544/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772476.074633:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534460544/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772476.074687:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772476.074696:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772476.074704:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534460544:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772476.074718:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534460672:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772476.074758:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772476.074778:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772476/real 1790772476] req@ffff9c4efac4dc00 x1877759534460672/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772476.074802:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534460672/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772476.074821:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772476.074826:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772476.074833:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534460672:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772476.136774:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772476.136790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772476.136792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772476.136804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772476.200764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772476.200806:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772476.200919:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772476.328765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772476.328793:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772476.392778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772476.456785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772476.520764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772476.520764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772476.520772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772476.584827:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772476.648779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772476.648784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772476.648826:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772476.776780:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772476.776792:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772476.776801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772476.776842:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772476.840757:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772476.840777:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772476.840794:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772476.840824:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772476.840834:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772476.840872:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772476.968762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772477.032754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772477.096765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772477.160772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772477.160783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772477.160804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772477.160815:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772477.224788:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772477.224809:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772477.224916:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772477.352804:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772477.352805:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772477.416774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772477.480796:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772477.544757:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772477.544757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772477.544786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772477.608765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772477.672783:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772477.672802:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772477.672808:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772477.800820:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772477.800831:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772477.800832:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772477.800892:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772477.864760:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772477.864769:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772477.864829:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772477.864832:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772477.864858:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772477.864859:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772477.992800:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772478.056747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772478.120757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772478.184792:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772478.184792:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772478.184793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772478.184803:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772478.248754:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772478.248767:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772478.248851:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772478.376746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772478.376758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772478.440788:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772478.504797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772478.568742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772478.568744:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772478.568792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772478.632760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772478.696829:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772478.696829:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772478.696830:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772478.824783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772478.824805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772478.824815:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772478.824858:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772478.888810:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772478.888845:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772478.888847:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772478.888859:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772478.888911:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772478.888949:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772479.016746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772479.080752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772479.144754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772479.208788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772479.208797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772479.208803:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772479.208805:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772479.272732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772479.272774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772479.272904:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772479.400775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772479.400782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772479.464770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772479.528775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772479.592744:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772479.592785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772479.592800:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772479.656760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772479.720774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772479.720801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772479.720865:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772479.848795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772479.848806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772479.848820:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772479.848823:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772479.912760:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772479.912789:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772479.912799:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772479.912807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772479.912838:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772479.912842:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772480.040728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772480.104766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772480.168777:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772480.232764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772480.232808:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772480.232853:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772480.232867:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772480.296818:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772480.296845:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772480.296905:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772480.424781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772480.424788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772480.488784:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772480.552762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772480.616765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772480.616766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772480.616793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772480.680769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772480.744759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772480.744788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772480.744870:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772480.872729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772480.872765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772480.872809:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772480.872813:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772480.936768:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772480.936787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772480.936796:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772480.936799:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772480.936851:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772480.936853:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772481.064796:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772481.128764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772481.192781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772481.192810:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772481.192821:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772481.192828:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772481.192843:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1477 00000100:00080000:0.0:1790772481.192859:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772481.192877:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772481.192939:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534460800/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772481.192982:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772481.192992:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772481.192995:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772481.193001:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1477 00000100:00080000:0.0:1790772481.193007:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772481.193014:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772481.193029:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772481.193048:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534460800:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772481.193053:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534460928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772481.193074:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772481.193079:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772481.193082:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772481.193088:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1477 00000100:00080000:0.0:1790772481.193092:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772481.193098:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772481.193105:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772481.193123:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534461056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772481.193138:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772481.193144:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772481.193147:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772481.193152:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1477 00000100:00080000:0.0:1790772481.193159:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772481.193190:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772481.193201:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772481.193210:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772481.193228:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534461184/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772481.193245:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772481.193250:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772481.193253:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772481.193259:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1477 00000100:00080000:0.0:1790772481.193266:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772481.193275:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772481.193283:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772481.193289:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772481/real 1790772481] req@ffff9c4ec7a7d180 x1877759534460800/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772481.193304:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534461312/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772481.193323:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772481.193331:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1790772481.193332:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534460800/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772481.193335:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772481.193352:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1477 00000100:00080000:0.0:1790772481.193360:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772481.193369:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772481.193370:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772481.193386:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1790772481.193388:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772481.193399:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534460800:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772481.193408:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534461440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772481.193424:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772481.193429:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772481.193433:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772481.193443:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534460928:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772481.193444:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1477 00000100:00080000:0.0:1790772481.193455:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772481.193462:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772481.193470:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772481.193490:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534461568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772481.193503:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772481.193522:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772481.193527:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772481.193533:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772481.193534:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772481/real 1790772481] req@ffff9c4ec7a7c700 x1877759534460928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772481.193547:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1477 00000100:00080000:0.0:1790772481.193560:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772481.193574:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772481.193582:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534460928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790772481.193585:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772481.193611:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534461696/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772481.193619:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772481.193628:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772481.193634:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772481.193640:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1790772481.193649:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534460928:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772481.193694:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534461056:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772481.193712:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772481.193729:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1477 00000800:00000100:3.0:1790772481.193730:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772481.193748:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772481.193759:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772481/real 1790772481] req@ffff9c4ec7a7fb80 x1877759534461056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772481.193767:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772481.193788:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772481.193797:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534461056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772481.193816:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790772481.193823:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534461824/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772481.193829:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772481.193849:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534461056:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772481.193868:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534461184:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772481.193916:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772481.193941:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772481/real 1790772481] req@ffff9c4ec7a7e300 x1877759534461184/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772481.193961:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534461184/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772481.193975:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772481.193980:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772481.193985:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534461184:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772481.193994:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534461312:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772481.194022:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772481.194036:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772481/real 1790772481] req@ffff9c4ec7a7ca80 x1877759534461312/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772481.194051:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534461312/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772481.194064:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772481.194068:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772481.194073:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534461312:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772481.194080:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534461440:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772481.194099:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772481.194111:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772481/real 1790772481] req@ffff9c4ec7a7ed80 x1877759534461440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772481.194135:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534461440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772481.194154:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772481.194158:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772481.194162:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534461440:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772481.194215:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534461568:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772481.194246:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772481.194261:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772481/real 1790772481] req@ffff9c4ec7a7d880 x1877759534461568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772481.194277:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534461568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772481.194288:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772481.194293:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772481.194298:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534461568:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772481.194306:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534461696:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772481.194326:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772481.194337:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772481/real 1790772481] req@ffff9c4ec7a7c380 x1877759534461696/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772481.194352:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534461696/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772481.194375:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772481.194379:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772481.194384:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534461696:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772481.194391:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534461824:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772481.194410:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772481.194422:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772481/real 1790772481] req@ffff9c4ec7a7f480 x1877759534461824/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772481.194436:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534461824/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772481.194449:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772481.194453:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772481.194458:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534461824:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772481.256753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772481.256795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772481.256834:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772481.256835:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772481.320789:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772481.320811:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772481.320830:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772481.448760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772481.448798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772481.512713:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772481.576819:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772481.640768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772481.640770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772481.640794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772481.704829:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772481.768763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772481.768769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772481.768770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772481.896776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772481.896778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772481.896810:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772481.896882:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772481.960733:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772481.960793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772481.960801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772481.960843:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772481.960844:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772481.960879:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772482.088761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772482.152808:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772482.216788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772482.280755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772482.280825:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772482.280831:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772482.280848:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772482.344825:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772482.344825:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772482.344965:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772482.472772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772482.472804:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772482.536838:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772482.600893:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772482.664806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772482.664811:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772482.664811:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772482.728772:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772482.792719:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772482.792779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772482.792806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772482.920780:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772482.920794:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772482.920798:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772482.920838:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772482.984770:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772482.984788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772482.984807:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772482.984849:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772482.984876:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772482.984879:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772483.112757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772483.176730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772483.240753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772483.304783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772483.304783:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772483.304865:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772483.304873:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772483.368802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772483.368806:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772483.368899:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772483.496770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772483.496775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772483.560743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772483.624762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772483.688762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772483.688791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772483.688806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772483.752804:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772483.816787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772483.816840:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772483.816861:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772483.944854:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772483.944880:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772483.944938:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772483.944992:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772484.008748:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772484.008823:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772484.008844:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772484.008865:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772484.008869:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772484.008884:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772484.136766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772484.200758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772484.264761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772484.328779:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772484.328836:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772484.328836:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772484.328838:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772484.392809:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772484.392839:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772484.392925:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772484.520749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772484.520782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772484.584761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772484.648771:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772484.712796:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772484.712808:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772484.712829:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772484.776765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772484.840765:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772484.840847:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772484.840853:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772484.968840:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772484.969000:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772484.969021:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772484.969163:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772485.032796:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772485.032824:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772485.032826:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772485.032839:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772485.032882:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772485.032922:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772485.160783:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772485.224721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772485.288777:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772485.352778:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772485.352781:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772485.352818:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772485.352831:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772485.416796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772485.416889:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772485.417021:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772485.544756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772485.544800:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772485.608749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772485.672761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772485.736761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772485.736794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772485.736820:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772485.800789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772485.864767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772485.864803:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772485.864817:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772485.992805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772485.992862:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772485.992884:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772485.993122:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772486.056755:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772486.056822:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772486.056822:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772486.056844:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772486.056860:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772486.056923:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772486.184812:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772486.248789:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772486.312788:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772486.312803:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772486.312812:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772486.312831:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1482 00000100:00080000:0.0:1790772486.312853:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772486.312858:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772486.312876:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772486.312947:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534461952/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772486.313003:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772486.313012:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772486.313017:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772486.313028:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1482 00000100:00080000:0.0:1790772486.313038:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772486.313051:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772486.313063:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534461952:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790772486.313100:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772486.313131:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534462080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772486.313149:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772486.313156:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772486.313159:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772486.313165:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1482 00000100:00080000:0.0:1790772486.313172:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772486.313179:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772486.313186:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772486.313202:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534462208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772486.313217:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772486.313228:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1790772486.313229:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772486.313231:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772486.313243:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1482 00000100:00080000:0.0:1790772486.313248:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772486.313257:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772486.313265:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772486.313281:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534462336/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772486.313294:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772486.313311:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772486.313315:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772486.313324:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1482 00000100:00100000:3.0:1790772486.313326:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772486/real 1790772486] req@ffff9c4ec7a7df80 x1877759534461952/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772486.313330:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772486.313346:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772486.313361:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772486.313379:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534462464/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772486.313394:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534461952/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772486.313397:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772486.313412:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772486.313419:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772486.313435:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1482 00000100:00080000:3.0:1790772486.313440:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772486.313441:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772486.313452:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772486.313461:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790772486.313463:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772486.313480:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534461952:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772486.313484:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534462592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772486.313507:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772486.313512:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772486.313515:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772486.313521:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1482 00000100:00080000:0.0:1790772486.313526:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772486.313531:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772486.313539:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772486.313548:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534462080:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772486.313564:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534462720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772486.313581:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772486.313586:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772486.313591:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772486.313598:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1482 00000800:00000100:3.0:1790772486.313605:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772486.313609:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772486.313622:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772486.313630:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772486.313642:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772486/real 1790772486] req@ffff9c4ec7a7f480 x1877759534462080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772486.313685:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534462848/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772486.313701:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772486.313706:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772486.313709:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772486.313716:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1482 00000100:00080000:0.0:1790772486.313724:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772486.313735:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772486.313749:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772486.313770:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534462080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772486.313781:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534462976/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772486.313809:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772486.313819:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772486.313827:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534462080:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772486.313856:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534462208:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772486.313902:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772486.313920:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772486/real 1790772486] req@ffff9c4ec7a7c380 x1877759534462208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772486.313946:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534462208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772486.313959:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772486.313964:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772486.313968:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534462208:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772486.313977:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534462336:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772486.313999:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772486.314012:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772486/real 1790772486] req@ffff9c4ec7a7d880 x1877759534462336/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772486.314028:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534462336/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772486.314051:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772486.314056:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772486.314061:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534462336:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772486.314096:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534462464:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772486.314138:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772486.314154:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772486/real 1790772486] req@ffff9c4ec7a7ed80 x1877759534462464/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772486.314169:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534462464/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772486.314191:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772486.314196:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772486.314201:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534462464:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772486.314208:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534462592:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772486.314229:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772486.314241:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772486/real 1790772486] req@ffff9c4ec7a7ca80 x1877759534462592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772486.314256:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534462592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772486.314277:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772486.314281:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772486.314285:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534462592:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772486.314312:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534462720:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772486.314335:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772486.314347:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772486/real 1790772486] req@ffff9c4ec7a7e300 x1877759534462720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772486.314363:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534462720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772486.314373:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772486.314377:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772486.314383:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534462720:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772486.314391:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534462848:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772486.314410:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772486.314422:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772486/real 1790772486] req@ffff9c4ec7a7fb80 x1877759534462848/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772486.314449:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534462848/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772486.314471:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772486.314479:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772486.314486:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534462848:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772486.314499:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534462976:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772486.314540:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772486.314561:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772486/real 1790772486] req@ffff9c4ec7a7c700 x1877759534462976/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772486.314577:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534462976/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772486.314590:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772486.314595:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772486.314600:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534462976:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772486.376785:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772486.376791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772486.376846:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772486.376886:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772486.440788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772486.440845:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772486.441037:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772486.568755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772486.568790:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772486.632750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772486.696810:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772486.760772:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772486.760779:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772486.760828:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772486.824866:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772486.888741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772486.888861:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772486.888869:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772487.016826:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772487.016927:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772487.016983:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772487.017025:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772487.080780:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772487.080807:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772487.080849:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772487.080856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772487.080863:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772487.080892:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772487.208751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772487.272751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772487.336756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772487.400788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772487.400795:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772487.400819:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772487.400819:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772487.464761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772487.464819:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772487.465030:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772487.592797:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772487.592799:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772487.656760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772487.720785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772487.784747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772487.784793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772487.784801:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772487.848782:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772487.912793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772487.912803:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772487.912804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772488.040852:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772488.040867:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772488.040867:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772488.040871:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772488.104785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772488.104792:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772488.104819:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772488.104851:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772488.104885:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772488.104915:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772488.232754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772488.296780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772488.360778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772488.424738:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772488.424778:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772488.424817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772488.424827:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772488.488832:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772488.488835:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772488.488841:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772488.616792:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772488.616792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772488.680739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772488.744830:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772488.808908:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772488.808909:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772488.808910:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772488.872748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772488.936759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772488.936764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772488.936799:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772489.064766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772489.064794:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772489.064808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772489.064833:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772489.128775:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772489.128812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772489.128831:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772489.128851:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772489.128856:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772489.128868:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772489.256790:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772489.320760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772489.384723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772489.448809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772489.448816:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772489.448817:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772489.448817:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772489.512783:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772489.512786:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772489.512792:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772489.640711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772489.640712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772489.704696:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772489.768777:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772489.832728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772489.832743:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772489.832751:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772489.896708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772489.960773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772489.960779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772489.960798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772490.088738:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772490.088763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772490.088785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772490.088840:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772490.152731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772490.152731:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772490.152799:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772490.152812:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772490.152813:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772490.152815:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772490.280779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772490.344721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772490.408762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772490.472736:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772490.472770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772490.472819:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772490.472831:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772490.536802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772490.536862:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772490.536941:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772490.664772:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772490.664774:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772490.728722:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772490.792787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772490.856785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772490.856792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772490.856827:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772490.920749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772490.984740:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772490.984765:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772490.984793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772491.112754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772491.112796:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772491.112798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772491.112798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772491.176731:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772491.176734:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772491.176795:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772491.176816:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772491.176819:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772491.176819:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772491.304725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772491.368743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772491.432733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772491.432767:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772491.432774:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772491.432777:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772491.432787:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1487 00000100:00080000:1.0:1790772491.432796:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772491.432806:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772491.432841:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534463104/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772491.432870:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772491.432873:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772491.432875:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772491.432879:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1487 00000100:00080000:1.0:1790772491.432882:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772491.432886:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772491.432895:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772491.432908:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534463232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772491.432918:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772491.432925:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534463104:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772491.432926:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772491.432931:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772491.432952:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1487 00000100:00080000:1.0:1790772491.432959:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772491.432982:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772491.433002:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772491.433022:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534463360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772491.433034:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772491.433037:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772491.433039:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772491.433044:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1487 00000100:00080000:1.0:1790772491.433050:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772491.433067:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772491.433073:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772491.433081:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772491.433086:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534463488/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772491.433095:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772491.433099:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772491.433101:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772491.433107:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1487 00000100:00080000:1.0:1790772491.433111:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772491.433115:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772491.433119:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772491.433131:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534463616/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772491.433140:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772491.433143:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772491/real 1790772491] req@ffff9c4efac4df80 x1877759534463104/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772491.433145:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772491.433150:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772491.433163:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1487 00000100:00080000:1.0:1790772491.433175:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772491.433180:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772491.433187:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772491.433192:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534463104/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772491.433208:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790772491.433214:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534463744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772491.433215:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772491.433230:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534463104:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772491.433253:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772491.433257:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772491.433259:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772491.433265:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1487 00000100:00080000:1.0:1790772491.433269:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772491.433278:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534463232:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772491.433279:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772491.433294:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772491.433311:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534463872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772491.433324:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1790772491.433324:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772491.433336:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772491.433339:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772491.433344:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1487 00000100:00080000:1.0:1790772491.433348:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772491.433363:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772491.433364:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772491/real 1790772491] req@ffff9c4efac4dc00 x1877759534463232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772491.433369:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790772491.433386:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534464000/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772491.433395:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772491.433397:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534463232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772491.433401:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772491.433409:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772491.433425:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1487 00000100:00080000:1.0:1790772491.433433:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772491.433447:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772491.433455:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:3.0:1790772491.433463:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790772491.433469:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534464128/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772491.433474:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772491.433479:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534463232:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772491.433501:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534463360:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772491.433531:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772491.433544:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772491/real 1790772491] req@ffff9c4efac4f800 x1877759534463360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772491.433558:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534463360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772491.433569:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772491.433574:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772491.433578:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534463360:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772491.433586:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534463488:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772491.433608:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772491.433618:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772491/real 1790772491] req@ffff9c4efac4f480 x1877759534463488/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772491.433626:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534463488/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772491.433635:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772491.433638:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772491.433640:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534463488:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772491.433646:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534463616:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772491.433658:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772491.433665:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772491/real 1790772491] req@ffff9c4efac4e300 x1877759534463616/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772491.433704:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534463616/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772491.433712:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772491.433715:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772491.433718:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534463616:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772491.433723:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534463744:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772491.433738:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772491.433747:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772491/real 1790772491] req@ffff9c4efac4c700 x1877759534463744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772491.433755:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534463744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772491.433769:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772491.433772:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772491.433774:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534463744:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772491.433791:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534463872:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772491.433812:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772491.433824:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772491/real 1790772491] req@ffff9c4efac4ed80 x1877759534463872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772491.433838:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534463872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772491.433856:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772491.433876:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772491.433880:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534463872:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772491.433888:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534464000:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772491.433914:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772491.433926:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772491/real 1790772491] req@ffff9c4efac4ea00 x1877759534464000/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772491.433939:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534464000/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772491.433951:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772491.433955:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772491.433959:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534464000:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772491.434003:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534464128:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772491.434021:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772491.434029:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772491/real 1790772491] req@ffff9c4efac4e680 x1877759534464128/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772491.434038:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534464128/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772491.434045:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772491.434048:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772491.434068:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534464128:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772491.496739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772491.496749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772491.496759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772491.496773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772491.560827:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772491.560899:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772491.560926:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772491.688788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772491.688809:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772491.752769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772491.816729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772491.880726:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772491.880732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772491.880779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772491.944730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772492.008741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772492.008788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772492.008789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772492.136774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772492.136821:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772492.136826:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772492.136828:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772492.200750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772492.200865:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772492.200874:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772492.200876:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772492.200916:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772492.200943:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772492.328778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772492.392774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772492.456763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772492.520854:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772492.520863:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772492.520882:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772492.520892:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772492.584833:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772492.584899:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772492.585022:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772492.712792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772492.712873:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772492.776759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772492.840800:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772492.904814:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772492.904818:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772492.904824:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772492.968752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772493.032773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772493.032803:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772493.032808:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772493.160823:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772493.160872:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772493.160915:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772493.160923:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772493.224739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772493.224754:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772493.224818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772493.224826:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772493.224872:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772493.224894:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772493.352801:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772493.416831:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772493.480784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772493.544780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772493.544781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772493.544849:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772493.544850:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772493.608846:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772493.608951:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772493.608985:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772493.736788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772493.736796:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772493.800765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772493.864784:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772493.928779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772493.928809:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772493.929729:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772493.992759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772494.056734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772494.056750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772494.056800:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772494.184774:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772494.184777:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772494.184789:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772494.184824:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772494.248749:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772494.248787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772494.248790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772494.248792:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772494.248816:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772494.248851:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772494.376766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772494.440773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772494.504735:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772494.568758:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772494.568773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772494.568789:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772494.568812:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772494.632776:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772494.632804:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772494.632893:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772494.760781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772494.760794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772494.824772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772494.888819:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772494.952743:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772494.952755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772494.952782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772495.016777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772495.080748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772495.080803:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772495.080809:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772495.208819:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772495.208819:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772495.208843:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772495.208844:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772495.272773:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772495.272830:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772495.272834:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772495.272835:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772495.272846:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772495.272932:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772495.400737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772495.464792:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772495.528784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772495.592746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772495.592757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772495.592758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772495.592763:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772495.656782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772495.656875:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772495.656963:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772495.784720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772495.784755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772495.848736:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772495.912780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772495.976801:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772495.976801:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772495.976804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772496.040817:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772496.104784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772496.104785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772496.104790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772496.232756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772496.232793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772496.232797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772496.232804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772496.296790:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772496.296800:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772496.296809:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772496.296823:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772496.296867:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772496.296920:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772496.424735:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772496.488786:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772496.552766:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772496.552812:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772496.552820:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772496.552827:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772496.552856:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1492 00000100:00080000:2.0:1790772496.552904:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772496.552925:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772496.553004:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534464256/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772496.553056:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772496.553062:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772496.553066:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772496.553076:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1492 00000100:00080000:2.0:1790772496.553085:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772496.553097:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772496.553117:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772496.553133:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534464256:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772496.553152:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534464384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772496.553173:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772496.553179:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772496.553182:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772496.553189:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1492 00000100:00080000:2.0:1790772496.553195:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772496.553223:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772496.553231:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1790772496.553244:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772496.553253:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534464512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772496.553270:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772496.553275:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772496.553278:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772496.553285:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1492 00000100:00080000:2.0:1790772496.553290:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772496.553297:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772496.553306:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772496.553316:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772496/real 1790772496] req@ffff9c4efbcc1500 x1877759534464256/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772496.553339:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534464640/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772496.553348:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534464256/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772496.553387:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772496.553390:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772496.553399:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772496.553403:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1790772496.553410:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534464256:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772496.553413:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772496.553426:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1492 00000100:00080000:2.0:1790772496.553434:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772496.553442:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772496.553450:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534464384:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790772496.553453:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772496.553489:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534464768/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772496.553505:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772496.553526:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772496.553534:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772496/real 1790772496] req@ffff9c4efbcc2a00 x1877759534464384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772496.553537:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772496.553546:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772496.553561:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1492 00000100:00080000:2.0:1790772496.553569:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772496.553573:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534464384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772496.553583:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772496.553599:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1790772496.553614:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772496.553625:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772496.553626:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534464896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772496.553637:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534464384:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772496.553691:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534464512:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772496.553706:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772496.553712:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772496.553716:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772496.553727:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1492 00000800:00000100:3.0:1790772496.553733:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772496.553737:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772496.553749:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772496.553758:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772496.553758:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772496/real 1790772496] req@ffff9c4efbcc0380 x1877759534464512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772496.553784:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534465024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772496.553787:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534464512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772496.553829:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772496.553831:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772496.553842:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772496.553845:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772496.553849:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772496.553857:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534464512:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772496.553863:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1492 00000100:00080000:2.0:1790772496.553897:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772496.553904:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534464640:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772496.553910:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772496.553922:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:3.0:1790772496.553948:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772496.553952:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534465152/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772496.553969:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772496/real 1790772496] req@ffff9c4efbcc3100 x1877759534464640/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772496.553981:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772496.553991:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1790772496.553993:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534464640/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772496.554001:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772496.554019:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772496.554021:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1492 00000100:00080000:3.0:1790772496.554029:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772496.554040:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772496.554040:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534464640:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772496.554054:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772496.554066:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772496.554085:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534464768:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772496.554097:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534465280/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772496.554117:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772496.554141:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772496/real 1790772496] req@ffff9c4efbcc0a80 x1877759534464768/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772496.554162:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534464768/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772496.554175:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772496.554180:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772496.554184:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534464768:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772496.554191:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534464896:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772496.554210:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772496.554222:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772496/real 1790772496] req@ffff9c4efbcc0700 x1877759534464896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772496.554236:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534464896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772496.554256:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772496.554260:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772496.554264:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534464896:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772496.554271:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534465024:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772496.554289:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772496.554300:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772496/real 1790772496] req@ffff9c4efbcc1f80 x1877759534465024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772496.554314:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534465024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772496.554326:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772496.554330:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772496.554334:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534465024:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772496.554340:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534465152:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772496.554358:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772496.554368:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772496/real 1790772496] req@ffff9c4efbcc1880 x1877759534465152/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772496.554381:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534465152/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772496.554395:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772496.554398:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772496.554402:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534465152:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772496.554422:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534465280:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772496.554442:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772496.554452:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772496/real 1790772496] req@ffff9c4efbcc2300 x1877759534465280/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772496.554467:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534465280/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772496.554478:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772496.554482:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772496.554487:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534465280:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772496.616773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772496.616797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772496.616799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772496.616801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772496.680777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772496.680778:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772496.680850:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772496.808730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772496.808749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772496.872770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772496.936845:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772497.000708:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772497.000716:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772497.000733:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772497.064718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772497.128739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772497.128783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772497.128792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772497.256772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772497.256787:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772497.256788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772497.256827:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772497.320769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772497.320823:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772497.320828:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772497.320829:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772497.320896:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772497.320904:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772497.448766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772497.512785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772497.576787:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772497.640759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772497.640777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772497.640795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772497.640796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772497.704812:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772497.704817:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772497.705012:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772497.832815:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772497.832830:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772497.896787:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772497.960753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772498.024753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772498.024792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772498.024809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772498.088811:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772498.152783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772498.152789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772498.152813:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772498.280752:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772498.280777:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772498.280792:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772498.280794:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772498.344800:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772498.344808:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772498.344819:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772498.344823:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772498.344921:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772498.344924:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772498.472770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772498.536729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772498.600750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772498.664767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772498.664769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772498.664796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772498.664798:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772498.728914:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772498.728942:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772498.728997:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772498.856778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772498.856801:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772498.920766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772498.984765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772499.048758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772499.048793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772499.048795:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772499.112773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772499.176795:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772499.176807:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772499.176840:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772499.304801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772499.304837:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772499.304845:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772499.304850:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772499.368790:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772499.368795:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772499.368804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772499.368830:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772499.368900:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772499.368911:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772499.496753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772499.560777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772499.624743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772499.688844:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772499.688844:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772499.688847:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772499.688872:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00010000:00010000:1.0:1790772499.752870:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772499.752883:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772499.752891:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772499.880763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772499.880799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772499.944771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772500.008793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772500.072872:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772500.072883:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772500.072910:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772500.136844:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772500.200779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772500.200876:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772500.200889:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772500.328845:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772500.328868:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772500.328914:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772500.328931:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772500.392778:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772500.392858:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772500.392878:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772500.392923:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772500.392923:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772500.392938:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772500.520873:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772500.584865:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772500.648737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772500.712824:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772500.712827:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772500.712842:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772500.712911:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772500.776877:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772500.776897:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772500.777029:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772500.904851:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772500.904856:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772500.968765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772501.032798:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772501.096839:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772501.096853:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772501.096854:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772501.160758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772501.224886:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772501.224891:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772501.224896:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772501.352751:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772501.352767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772501.352811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772501.352880:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772501.416826:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772501.416849:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772501.416893:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772501.416893:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772501.416929:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772501.416941:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772501.544867:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772501.608749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772501.672796:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772501.672821:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772501.672832:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772501.672839:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772501.672857:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1497 00000100:00080000:1.0:1790772501.672875:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772501.672895:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772501.672959:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534465408/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772501.673003:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772501.673009:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772501.673012:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772501.673018:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1497 00000100:00080000:1.0:1790772501.673024:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772501.673031:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772501.673048:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772501.673076:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534465536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772501.673078:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534465408:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772501.673107:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772501.673112:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772501.673115:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772501.673122:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1497 00000100:00080000:1.0:1790772501.673127:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772501.673133:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772501.673139:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772501.673157:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534465664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772501.673170:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772501.673176:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772501.673179:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772501.673185:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1497 00000100:00080000:1.0:1790772501.673191:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772501.673197:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:3.0:1790772501.673206:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772501.673207:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772501.673238:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534465792/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772501.673260:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772501.673269:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772501.673274:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772501.673279:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772501/real 1790772501] req@ffff9c4efac4fb80 x1877759534465408/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772501.673291:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1497 00000100:00080000:1.0:1790772501.673308:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772501.673324:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534465408/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772501.673328:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772501.673343:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:3.0:1790772501.673353:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772501.673363:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790772501.673371:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534465920/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772501.673374:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534465408:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772501.673401:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772501.673407:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772501.673410:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772501.673419:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1497 00000100:00100000:3.0:1790772501.673426:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534465536:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772501.673427:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772501.673443:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772501.673452:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772501.673472:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534466048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772501.673483:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772501.673502:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772501.673508:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772501.673515:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772501.673516:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772501/real 1790772501] req@ffff9c4efac4e680 x1877759534465536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772501.673528:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1497 00000100:00080000:1.0:1790772501.673539:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772501.673552:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772501.673560:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534465536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772501.673562:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772501.673589:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534466176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772501.673599:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772501.673604:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772501.673612:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772501.673613:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772501.673619:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772501.673627:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534465536:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772501.673633:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1497 00000100:00100000:3.0:1790772501.673695:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534465664:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772501.673701:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772501.673713:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772501.673725:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:3.0:1790772501.673735:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772501.673755:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772501/real 1790772501] req@ffff9c4efac4ea00 x1877759534465664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772501.673763:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534466304/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772501.673818:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534465664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772501.673826:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772501.673837:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772501.673841:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772501.673850:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772501.673852:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772501.673860:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534465664:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772501.673871:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1497 00000100:00100000:3.0:1790772501.673877:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534465792:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790772501.673886:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772501.673903:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:3.0:1790772501.673914:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772501.673917:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772501.673933:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772501/real 1790772501] req@ffff9c4efac4ed80 x1877759534465792/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772501.673952:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534466432/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772501.673955:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534465792/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772501.674017:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772501.674027:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772501.674035:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534465792:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772501.674045:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534465920:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772501.674069:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772501.674082:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772501/real 1790772501] req@ffff9c4efac4c700 x1877759534465920/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772501.674098:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534465920/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772501.674111:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772501.674115:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772501.674119:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534465920:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772501.674141:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534466048:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772501.674163:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772501.674176:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772501/real 1790772501] req@ffff9c4efac4e300 x1877759534466048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772501.674191:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534466048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772501.674212:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772501.674216:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772501.674220:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534466048:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772501.674228:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534466176:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772501.674246:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772501.674258:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772501/real 1790772501] req@ffff9c4efac4f480 x1877759534466176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772501.674273:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534466176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772501.674293:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772501.674298:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772501.674303:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534466176:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772501.674309:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534466304:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772501.674328:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772501.674339:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772501/real 1790772501] req@ffff9c4efac4f800 x1877759534466304/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772501.674353:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534466304/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772501.674366:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772501.674370:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772501.674375:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534466304:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772501.674383:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534466432:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772501.674402:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772501.674413:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772501/real 1790772501] req@ffff9c4efac4dc00 x1877759534466432/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772501.674428:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534466432/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772501.674440:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772501.674445:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772501.674450:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534466432:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772501.736808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772501.736832:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772501.736846:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772501.736864:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772501.800816:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772501.800896:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772501.800898:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772501.928798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772501.928834:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772501.992749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772502.056823:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772502.120789:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772502.120790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772502.120836:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772502.184747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772502.248753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772502.248806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772502.248808:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772502.376788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772502.376794:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772502.376812:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772502.376836:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772502.440788:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772502.440805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772502.440829:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772502.440844:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772502.440867:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772502.440885:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772502.568815:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772502.632803:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772502.696780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772502.760792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772502.760807:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772502.760831:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772502.760834:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772502.824740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772502.824840:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772502.824979:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772502.952775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772502.952839:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772503.016805:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772503.080762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772503.144781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772503.144852:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772503.144861:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772503.208813:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772503.272774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772503.272820:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772503.272825:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772503.400793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772503.400796:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772503.400810:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772503.400832:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772503.464792:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772503.464812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772503.464825:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772503.464845:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772503.464867:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772503.464889:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772503.592841:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772503.656767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772503.720795:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772503.784785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772503.784817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772503.784837:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772503.784846:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772503.848829:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772503.848873:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772503.848895:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772503.976804:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772503.976812:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772504.040812:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772504.104874:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772504.168802:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772504.168806:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772504.168806:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772504.232783:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772504.296786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772504.296822:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772504.296837:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772504.424767:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772504.424767:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772504.424767:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772504.424768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772504.488787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772504.488793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772504.488824:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772504.488831:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772504.488838:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772504.488864:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772504.616800:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772504.680807:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772504.744806:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772504.808773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772504.808820:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772504.808822:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772504.808834:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772504.872836:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772504.872889:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772504.873040:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772505.000844:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772505.000844:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772505.064782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772505.128825:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772505.192785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772505.192797:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772505.192830:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772505.256825:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772505.320838:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772505.320883:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772505.320884:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772505.448823:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772505.448831:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772505.448840:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772505.448880:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772505.512786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772505.512803:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772505.512805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772505.512836:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772505.512866:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772505.512890:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772505.640802:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772505.704815:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772505.768761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772505.832778:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772505.832825:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772505.832860:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772505.832864:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772505.896838:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772505.896895:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772505.897036:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772506.024771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772506.024812:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772506.088799:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772506.152815:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772506.216822:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772506.216832:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772506.216833:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772506.280788:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772506.344771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772506.344788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772506.344805:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772506.472782:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772506.472796:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772506.472803:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772506.472810:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772506.536783:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772506.536813:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772506.536861:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772506.536885:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772506.536888:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772506.536937:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772506.664779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772506.728781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772506.792811:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772506.792826:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772506.792842:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772506.792851:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772506.792872:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1502 00000100:00080000:3.0:1790772506.792895:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772506.792920:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772506.792991:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534466560/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772506.793040:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772506.793046:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772506.793049:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772506.793055:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1502 00000100:00080000:3.0:1790772506.793062:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772506.793072:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772506.793089:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772506.793110:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534466560:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772506.793113:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534466688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772506.793146:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772506.793151:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772506.793153:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772506.793159:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1502 00000100:00080000:3.0:1790772506.793163:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772506.793169:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772506.793177:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772506.793193:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534466816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772506.793206:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772506.793212:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772506.793215:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772506.793220:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1502 00000100:00080000:3.0:1790772506.793226:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772506.793234:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772506.793241:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772506.793250:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772506.793268:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534466944/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772506.793280:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772506.793285:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772506.793288:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772506.793296:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1502 00000100:00100000:2.0:1790772506.793311:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772506/real 1790772506] req@ffff9c4ef73c0e00 x1877759534466560/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772506.793318:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772506.793336:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772506.793345:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772506.793354:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534466560/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772506.793368:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534467072/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772506.793381:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772506.793388:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772506.793395:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772506.793398:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772506.793405:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772506.793411:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534466560:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772506.793417:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1502 00000100:00080000:3.0:1790772506.793427:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772506.793435:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772506.793442:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772506.793456:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534467200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772506.793463:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534466688:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772506.793486:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772506.793492:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772506.793495:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772506.793504:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1502 00000100:00080000:3.0:1790772506.793511:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772506.793515:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772506.793523:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772506.793533:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772506.793543:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772506/real 1790772506] req@ffff9c4ef73c0000 x1877759534466688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772506.793563:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534467328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772506.793567:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534466688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772506.793632:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772506.793639:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772506.793642:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772506.793724:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1790772506.793726:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534466688:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772506.793731:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772506.793750:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1502 00000100:00100000:2.0:1790772506.793759:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534466816:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772506.793760:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772506.793775:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772506.793784:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772506.793809:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534467456/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772506.793825:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772506.793833:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772506.793844:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772506.793849:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772506.793852:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772506/real 1790772506] req@ffff9c4ef73c1c00 x1877759534466816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772506.793868:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1502 00000100:00080000:3.0:1790772506.793883:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772506.793885:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534466816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772506.793903:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772506.793908:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790772506.793920:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:2.0:1790772506.793923:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772506.793937:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534466816:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772506.793951:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534467584/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772506.793959:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534466944:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772506.794020:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772506.794040:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772506/real 1790772506] req@ffff9c4ef73c2a00 x1877759534466944/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772506.794057:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534466944/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772506.794072:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772506.794077:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772506.794082:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534466944:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772506.794090:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534467072:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772506.794115:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772506.794127:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772506/real 1790772506] req@ffff9c4ef73c0700 x1877759534467072/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772506.794152:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534467072/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772506.794165:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772506.794169:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772506.794174:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534467072:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772506.794204:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534467200:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772506.794226:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772506.794239:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772506/real 1790772506] req@ffff9c4ef73c2300 x1877759534467200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772506.794254:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534467200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772506.794276:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772506.794281:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772506.794285:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534467200:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772506.794293:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534467328:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772506.794312:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772506.794324:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772506/real 1790772506] req@ffff9c4ef73c1500 x1877759534467328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772506.794339:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534467328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772506.794358:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772506.794363:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772506.794368:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534467328:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772506.794375:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534467456:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772506.794393:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772506.794405:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772506/real 1790772506] req@ffff9c4ef73c0a80 x1877759534467456/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772506.794419:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534467456/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772506.794434:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772506.794437:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772506.794442:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534467456:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772506.794450:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534467584:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772506.794469:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772506.794481:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772506/real 1790772506] req@ffff9c4ef73c3800 x1877759534467584/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772506.794496:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534467584/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772506.794508:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772506.794512:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772506.794518:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534467584:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772506.856764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772506.856797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772506.856803:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772506.856842:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772506.920804:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772506.920922:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772506.920944:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772507.048797:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772507.048817:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772507.112751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772507.176815:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772507.240762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772507.240785:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772507.240823:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772507.304789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772507.368772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772507.368804:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772507.368824:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772507.496786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772507.496799:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772507.496828:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772507.496838:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772507.560845:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772507.560846:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772507.560876:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772507.560888:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772507.560928:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772507.560935:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772507.688777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772507.752762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772507.816784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772507.880798:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772507.880808:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772507.880823:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772507.880831:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772507.944804:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772507.944863:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772507.944928:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772508.072779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772508.072824:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772508.136772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772508.200809:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772508.264785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772508.264786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772508.264838:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772508.328774:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772508.392705:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772508.392729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772508.392810:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772508.520763:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772508.520820:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772508.520860:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772508.520865:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772508.584810:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772508.584820:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772508.584833:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772508.584845:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772508.584880:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772508.584918:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772508.712785:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772508.776781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772508.840744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772508.904753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772508.904758:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772508.904810:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772508.904811:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772508.968736:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772508.968777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772508.968861:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772509.096823:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772509.096827:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772509.160779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772509.224793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772509.288749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772509.288802:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772509.288856:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772509.352768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772509.416772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772509.416810:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772509.416834:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772509.544774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772509.544787:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772509.544793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772509.544821:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772509.608764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772509.608777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772509.608828:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772509.608840:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772509.608867:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772509.608895:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772509.736809:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772509.800760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772509.864771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772509.928757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772509.928764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772509.928787:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772509.928823:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772509.992852:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772509.992909:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772509.992995:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772510.120764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772510.120775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772510.184825:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772510.248758:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772510.312760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772510.312872:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772510.312873:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772510.376769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772510.440768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772510.440797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772510.440799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772510.568755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772510.568777:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772510.568780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772510.568824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772510.632798:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772510.632839:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772510.632856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772510.632861:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772510.632905:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772510.632911:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772510.760764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772510.824776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772510.888750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772510.952797:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772510.952798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772510.952809:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772510.952811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772511.016753:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772511.016853:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772511.016856:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772511.144775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772511.144792:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772511.208773:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772511.272909:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772511.336768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772511.336787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772511.336812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772511.400768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772511.464777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772511.464832:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772511.464836:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772511.592737:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772511.592738:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772511.592799:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772511.592814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772511.656745:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772511.656764:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772511.656800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772511.656830:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772511.656839:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772511.656839:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772511.784769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772511.848772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772511.912779:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772511.912787:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772511.912796:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772511.912808:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772511.912830:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1507 00000100:00080000:3.0:1790772511.912852:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772511.912878:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772511.912960:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534467712/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772511.913007:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772511.913013:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772511.913016:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772511.913023:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1507 00000100:00080000:3.0:1790772511.913031:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772511.913043:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772511.913062:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772511.913078:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534467712:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772511.913090:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534467840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772511.913112:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772511.913118:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772511.913121:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772511.913131:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1507 00000100:00080000:3.0:1790772511.913138:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772511.913160:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772511.913171:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790772511.913191:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772511.913193:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534467968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772511.913214:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772511.913220:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772511.913224:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772511.913230:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1507 00000100:00080000:3.0:1790772511.913236:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772511.913246:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772511.913258:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772511.913262:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772511/real 1790772511] req@ffff9c4ef73c2d80 x1877759534467712/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772511.913280:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534468096/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772511.913296:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772511.913298:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534467712/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772511.913311:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772511.913320:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772511.913349:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1507 00000100:00080000:2.0:1790772511.913360:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772511.913361:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772511.913379:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772511.913390:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:3.0:1790772511.913391:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772511.913415:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534467712:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772511.913418:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534468224/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772511.913458:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772511.913464:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772511.913467:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772511.913478:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1507 00000100:00080000:3.0:1790772511.913482:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772511.913489:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772511.913497:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772511.913500:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534467840:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772511.913524:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534468352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772511.913544:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772511.913549:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772511.913552:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772511.913561:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1507 00000100:00080000:3.0:1790772511.913596:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772511.913609:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772511.913618:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772511.913636:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534468480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772511.913649:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772511.913690:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772511.913693:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772511.913699:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1507 00000100:00080000:3.0:1790772511.913705:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772511.913712:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772511.913720:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772511.913737:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534468608/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772511.913751:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772511.913754:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772511.913766:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772511.913772:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772511.913782:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1507 00000100:00080000:3.0:1790772511.913792:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772511.913803:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772511.913812:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772511/real 1790772511] req@ffff9c4ef73c3800 x1877759534467840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772511.913822:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772511.913877:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534467840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772511.913879:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534468736/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772511.913960:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772511.913968:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772511.913975:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534467840:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772511.913989:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534467968:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772511.914026:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772511.914041:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772511/real 1790772511] req@ffff9c4ef73c0a80 x1877759534467968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772511.914058:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534467968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772511.914070:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772511.914075:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772511.914080:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534467968:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772511.914088:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534468096:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772511.914109:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772511.914122:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772511/real 1790772511] req@ffff9c4ef73c1500 x1877759534468096/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772511.914136:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534468096/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772511.914158:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772511.914162:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772511.914167:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534468096:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772511.914175:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534468224:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772511.914195:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772511.914207:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772511/real 1790772511] req@ffff9c4ef73c2300 x1877759534468224/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772511.914221:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534468224/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772511.914233:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772511.914237:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772511.914242:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534468224:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772511.914272:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534468352:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772511.914295:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772511.914309:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772511/real 1790772511] req@ffff9c4ef73c0700 x1877759534468352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772511.914324:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534468352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772511.914343:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772511.914348:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772511.914354:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534468352:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772511.914362:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534468480:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772511.914394:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772511.914416:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772511/real 1790772511] req@ffff9c4ef73c2a00 x1877759534468480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772511.914442:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534468480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772511.914462:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772511.914469:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772511.914477:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534468480:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772511.914491:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534468608:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772511.914524:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772511.914548:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772511/real 1790772511] req@ffff9c4ef73c1c00 x1877759534468608/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772511.914608:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534468608/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772511.914630:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772511.914638:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772511.914646:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534468608:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772511.914691:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534468736:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772511.914749:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772511.914768:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772511/real 1790772511] req@ffff9c4ef73c0000 x1877759534468736/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772511.914785:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534468736/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772511.914798:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772511.914802:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772511.914807:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534468736:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772511.976811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772511.976827:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772511.976850:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772511.976854:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772512.040841:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772512.040849:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772512.040981:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772512.168728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772512.168759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772512.232750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772512.296778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772512.360824:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772512.360833:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772512.360837:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772512.424786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772512.488789:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772512.488835:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772512.488846:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772512.616815:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772512.616816:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772512.616843:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772512.616843:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772512.680818:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772512.680837:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772512.680840:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772512.680840:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772512.680874:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772512.680927:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772512.808769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772512.872768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772512.936808:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772513.000736:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772513.000763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772513.000765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772513.000823:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772513.064765:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772513.064877:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772513.064899:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772513.192787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772513.192796:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772513.256782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772513.320830:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772513.384773:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772513.384857:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772513.384859:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772513.448771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772513.512786:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772513.512797:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772513.512854:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772513.640774:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772513.640776:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772513.640788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772513.640801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772513.704824:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772513.704846:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772513.704847:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772513.704858:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772513.704913:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772513.705747:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772513.832781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772513.896785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772513.960724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772514.024750:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772514.024792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772514.024794:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772514.024795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772514.088811:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772514.088829:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772514.088953:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772514.216721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772514.216749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772514.280740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772514.344735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772514.408707:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772514.408753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772514.408760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772514.472736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772514.536725:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772514.536753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772514.536760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772514.664734:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772514.664734:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772514.664738:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772514.664745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772514.728726:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772514.728746:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772514.728803:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772514.728807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772514.728807:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772514.728821:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772514.856750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772514.920715:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772514.984752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772515.048764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772515.048767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772515.048780:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772515.048780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772515.112803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772515.112803:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772515.112887:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772515.240764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772515.240789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772515.304770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772515.368781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772515.432769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772515.432799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772515.432805:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772515.496778:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772515.560753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772515.560778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772515.560794:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772515.688806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772515.688818:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772515.688826:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772515.688829:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772515.752739:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772515.752797:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772515.752806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772515.752821:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772515.752829:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772515.752864:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772515.880777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772515.944746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772516.008748:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772516.072772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772516.072805:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772516.072810:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772516.072821:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772516.136782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772516.136856:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772516.136887:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772516.264765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772516.264775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772516.328771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772516.392769:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772516.456835:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772516.456850:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772516.456852:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772516.520765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772516.584741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772516.584793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772516.584812:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772516.712792:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772516.712818:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772516.712819:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772516.712832:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772516.776791:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772516.776805:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772516.776848:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772516.776859:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772516.776901:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772516.776931:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772516.904760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772516.968753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772517.032827:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772517.032841:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772517.032850:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772517.032871:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772517.032875:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1512 00000100:00080000:0.0:1790772517.032906:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772517.032935:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772517.033007:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534468864/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772517.033053:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772517.033059:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772517.033062:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772517.033069:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1512 00000100:00080000:0.0:1790772517.033075:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772517.033084:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772517.033113:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772517.033134:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534468992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772517.033150:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534468864:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772517.033162:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772517.033169:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772517.033173:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772517.033182:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1512 00000100:00080000:0.0:1790772517.033188:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772517.033199:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772517.033209:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772517.033230:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534469120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772517.033244:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772517.033252:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772517.033254:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772517.033261:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1512 00000800:00000100:2.0:1790772517.033271:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772517.033272:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772517.033294:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772517.033314:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772517.033337:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534469248/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772517.033352:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772517.033360:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1790772517.033363:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772517/real 1790772517] req@ffff9c4ec7a7d180 x1877759534468864/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772517.033365:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772517.033380:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1512 00000100:00080000:0.0:1790772517.033394:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772517.033407:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772517.033420:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772517.033443:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534469376/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772517.033454:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534468864/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772517.033511:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772517.033542:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772517.033551:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772517.033552:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772517.033572:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772517.033580:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1512 00000100:00100000:2.0:1790772517.033590:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534468864:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772517.033594:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772517.033616:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772517.033631:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772517.033649:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534468992:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772517.033738:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534469504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772517.033752:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772517.033781:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772517.033793:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772517/real 1790772517] req@ffff9c4ec7a7c700 x1877759534468992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772517.033795:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772517.033805:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772517.033834:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1512 00000100:00080000:0.0:1790772517.033851:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772517.033852:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534468992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772517.033879:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772517.033884:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772517.033893:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790772517.033897:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772517.033902:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534468992:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772517.033923:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534469120:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772517.033947:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534469632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772517.033967:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772517.033990:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772517.034001:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772517/real 1790772517] req@ffff9c4ec7a7fb80 x1877759534469120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772517.034004:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772517.034012:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772517.034037:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1512 00000100:00080000:0.0:1790772517.034050:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772517.034055:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534469120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772517.034064:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772517.034076:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:2.0:1790772517.034105:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790772517.034106:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534469760/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772517.034137:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772517.034141:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772517.034151:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772517.034156:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772517.034165:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534469120:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772517.034171:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1512 00000100:00080000:0.0:1790772517.034186:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772517.034200:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772517.034202:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534469248:192.168.204.150@tcp:8:kworker.0 00000008:00100000:0.0:1790772517.034213:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772517.034236:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534469888/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772517.034264:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772517.034289:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772517/real 1790772517] req@ffff9c4ec7a7e300 x1877759534469248/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772517.034306:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534469248/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772517.034320:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772517.034325:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772517.034330:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534469248:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772517.034343:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534469376:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772517.034383:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772517.034407:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772517/real 1790772517] req@ffff9c4ec7a7ca80 x1877759534469376/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772517.034433:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534469376/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772517.034450:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772517.034456:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772517.034461:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534469376:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772517.034521:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534469504:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772517.034554:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772517.034571:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772517/real 1790772517] req@ffff9c4ec7a7ed80 x1877759534469504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772517.034588:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534469504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772517.034607:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772517.034612:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772517.034617:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534469504:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772517.034625:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534469632:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772517.034645:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772517.034692:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772517/real 1790772517] req@ffff9c4ec7a7d880 x1877759534469632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772517.034709:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534469632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772517.034720:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772517.034725:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772517.034729:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534469632:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772517.034738:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534469760:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772517.034761:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772517.034774:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772517/real 1790772517] req@ffff9c4ec7a7c380 x1877759534469760/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772517.034810:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534469760/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772517.034833:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772517.034838:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772517.034842:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534469760:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772517.034851:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534469888:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772517.034874:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772517.034887:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772517/real 1790772517] req@ffff9c4ec7a7f480 x1877759534469888/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772517.034902:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534469888/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772517.034914:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772517.034919:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772517.034924:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534469888:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772517.096773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772517.096790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772517.096805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772517.096853:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772517.160784:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772517.160853:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772517.160946:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772517.288764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772517.288782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772517.352790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772517.416786:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772517.480790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772517.480791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772517.480799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772517.544805:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772517.608764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772517.608771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772517.608794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772517.736815:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772517.736858:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772517.736881:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772517.736897:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772517.800832:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772517.800846:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772517.800855:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772517.800877:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772517.800919:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772517.800943:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772517.928771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772517.992776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772518.056750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772518.120830:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772518.120837:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772518.120837:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772518.120884:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772518.184846:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772518.184917:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772518.184921:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772518.312797:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772518.312811:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772518.376760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772518.440889:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772518.504777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772518.504787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772518.504793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772518.568770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772518.632783:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772518.632783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772518.632821:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772518.760759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772518.760793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772518.760830:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772518.760909:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772518.824785:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772518.824819:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772518.824825:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772518.824838:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772518.824875:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772518.824909:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772518.952792:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772519.016760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772519.080742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772519.144760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772519.144821:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772519.144827:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772519.144827:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772519.208796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772519.208827:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772519.208899:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772519.336744:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772519.336771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772519.400749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772519.464778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772519.528794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772519.528797:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772519.528805:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772519.592766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772519.656765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772519.656776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772519.656790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772519.784762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772519.784774:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772519.784790:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772519.784804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772519.848770:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772519.848792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772519.848822:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772519.848828:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772519.848846:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772519.848866:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772519.976783:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772520.040744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772520.104743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772520.168778:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772520.168792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772520.168803:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772520.168803:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772520.232767:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772520.232812:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772520.232889:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772520.360769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772520.360774:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772520.424770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772520.488797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772520.552771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772520.552777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772520.552783:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772520.616756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772520.680799:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772520.680809:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772520.680812:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772520.808757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772520.808791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772520.808796:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772520.808797:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772520.872759:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772520.872799:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772520.872809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772520.872809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772520.872833:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772520.872862:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772521.000774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772521.064777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772521.128751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772521.192767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772521.192787:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772521.192797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772521.192813:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772521.256826:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772521.256842:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772521.256898:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772521.384771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772521.384780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772521.448758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772521.512770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772521.576786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772521.576788:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772521.576791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772521.640763:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772521.704773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772521.704783:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772521.704786:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772521.832785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772521.832826:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772521.832850:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772521.832853:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772521.896799:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772521.896811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772521.896859:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772521.896863:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772521.896891:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772521.896925:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772522.024759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772522.088759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772522.152771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772522.152839:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772522.152851:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772522.152875:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772522.152888:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1518 00000100:00080000:1.0:1790772522.152902:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772522.152915:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772522.152971:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534470016/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772522.153010:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772522.153019:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772522.153021:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772522.153026:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1518 00000100:00080000:1.0:1790772522.153033:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772522.153039:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772522.153052:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772522.153071:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534470144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772522.153085:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534470016:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772522.153106:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772522.153112:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772522.153115:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772522.153121:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1518 00000100:00080000:1.0:1790772522.153128:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772522.153136:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772522.153144:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772522.153160:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534470272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772522.153172:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772522.153177:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772522.153180:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772522.153186:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1518 00000100:00080000:1.0:1790772522.153192:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772522.153202:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772522.153212:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772522.153225:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534470400/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772522.153239:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772522.153243:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772522.153245:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772522.153251:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1518 00000100:00080000:1.0:1790772522.153255:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772522.153265:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772522.153271:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772522.153283:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534470528/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772522.153293:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772522.153297:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772522.153300:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772522.153307:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1518 00000100:00080000:1.0:1790772522.153315:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772522.153321:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534470144:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772522.153330:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772522.153343:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772522.153392:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534470656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772522.153408:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534470272:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772522.153441:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772522.153450:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772522.153456:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772522.153466:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534470400:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790772522.153473:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1518 00000100:00080000:1.0:1790772522.153488:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772522.153503:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772522.153515:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772522.153523:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534470528:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772522.153543:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534470784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772522.153578:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772522.153585:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772522.153589:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772522.153604:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1518 00000100:00100000:2.0:1790772522.153609:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534470656:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772522.153616:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772522.153651:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772522.153665:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772522.153742:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534470784:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772522.153757:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534470912/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772522.153809:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772522.153817:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772522.153823:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772522.153832:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1518 00000100:00080000:1.0:1790772522.153839:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772522.153843:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534470912:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790772522.153853:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772522.153882:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772522.153904:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534471040/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772522.153951:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534471040:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772522.216902:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772522.216906:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772522.216907:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772522.216908:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772522.280815:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772522.280882:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772522.280888:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772522.408784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772522.408784:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772522.472764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772522.536779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772522.600750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772522.600782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772522.600785:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772522.664779:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772522.728757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772522.728774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772522.728788:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772522.856757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772522.856782:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772522.856808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772522.856814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772522.920754:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772522.920761:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772522.920796:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772522.920803:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772522.920834:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772522.920837:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772523.048738:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772523.112750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772523.176766:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772523.240758:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772523.240782:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772523.240786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772523.240789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772523.304785:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772523.304820:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772523.304906:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772523.432761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772523.432808:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772523.496766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772523.560800:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772523.624758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772523.624768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772523.624794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772523.688737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772523.752761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772523.752774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772523.752782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772523.880762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772523.880770:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772523.880802:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772523.880803:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772523.944774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772523.944800:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772523.944806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772523.944815:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772523.944843:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772523.944850:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772524.072740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772524.136763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772524.200758:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772524.264776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772524.264792:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772524.264793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772524.264798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772524.328770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772524.328817:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772524.328883:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772524.456734:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772524.456765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772524.520760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772524.584705:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772524.648720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772524.648776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772524.648790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772524.712773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772524.776777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772524.776778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772524.776778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772524.904779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772524.904781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772524.904842:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772524.904843:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772524.968806:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772524.968812:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772524.968824:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772524.968838:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772524.968876:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772524.968906:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772525.096752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772525.160761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772525.224754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:0.0:1790772525.225058:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:0.0:1790772525.225121:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1790772525.225149:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772525.225227:0:2384:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:0.0:1790772525.225247:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772525.225261:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772525.225273:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772525.225286:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772525.225332:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772525.225358:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790772525.225365:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772522/real 1790772525] req@ffff9c4efac4e680 x1877759534471040/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1790772525.225384:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790772525.225442:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534471040/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772525.225474:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772525.225486:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772525.225497:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534471040:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772525.225520:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772522/real 1790772525] req@ffff9c4efac4ea00 x1877759534470912/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772525.225547:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534470912/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772525.225570:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772525.225578:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772525.225587:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534470912:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772525.225602:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772522/real 1790772525] req@ffff9c4efac4c700 x1877759534470656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772525.225630:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534470656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772525.225705:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772525.225718:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772525.225727:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534470656:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772525.225747:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772522/real 1790772525] req@ffff9c4efac4ed80 x1877759534470784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772525.225777:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534470784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772525.225797:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772525.225807:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772525.225814:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534470784:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772525.225831:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772522/real 1790772525] req@ffff9c4efac4dc00 x1877759534470144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772525.225860:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534470144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772525.225895:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772525.225907:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772525.225916:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534470144:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772525.225934:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772522/real 1790772525] req@ffff9c4efac4f800 x1877759534470272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772525.225962:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534470272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772525.225981:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772525.225989:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772525.225998:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534470272:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772525.226013:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772522/real 1790772525] req@ffff9c4efac4f480 x1877759534470400/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772525.226040:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534470400/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772525.226063:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772525.226070:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772525.226078:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534470400:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772525.226095:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772522/real 1790772525] req@ffff9c4efac4e300 x1877759534470528/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772525.226123:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534470528/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772525.226144:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772525.226153:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772525.226160:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534470528:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772525.226175:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772522/real 1790772525] req@ffff9c4efac4df80 x1877759534470016/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772525.226215:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534470016/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772525.226232:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772525.226240:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772525.226247:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534470016:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772525.288757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772525.288852:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772525.288863:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772525.288864:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772525.352797:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772525.352815:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772525.352973:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772525.480777:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772525.480822:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772525.544768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772525.608873:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772525.672754:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772525.672778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772525.672793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772525.736753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772525.800784:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772525.800796:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772525.800798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772525.928791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772525.928795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772525.928813:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772525.928816:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772525.992773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772525.992810:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772525.992822:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772525.992829:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772525.992872:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772525.992875:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772526.120759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772526.184759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772526.248736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772526.312768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772526.312789:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772526.312791:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772526.312792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772526.376774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772526.376795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772526.376899:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772526.504756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772526.504784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772526.568744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772526.632760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772526.696769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772526.696785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772526.696810:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772526.760766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772526.824755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772526.824783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772526.824791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772526.952760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772526.952773:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772526.952787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772526.952788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772527.016763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772527.016792:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772527.016803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772527.016803:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772527.016833:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772527.016871:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772527.144765:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772527.208770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772527.272741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772527.272886:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772527.272897:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772527.272905:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772527.272922:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1523 00000100:00080000:0.0:1790772527.272938:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772527.272958:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772527.273017:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534471168/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772527.273060:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772527.273066:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772527.273069:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772527.273075:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1523 00000100:00080000:0.0:1790772527.273081:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772527.273091:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772527.273112:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772527.273127:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534471168:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772527.273147:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534471296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772527.273172:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772527.273180:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772527.273184:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772527.273193:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1523 00000100:00080000:0.0:1790772527.273202:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772527.273214:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772527.273226:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772527.273252:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534471424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772527.273306:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772527.273306:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772527.273321:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772527.273325:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772527.273336:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1523 00000100:00080000:0.0:1790772527.273346:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772527.273370:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772527.273383:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772527.273406:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772527/real 1790772527] req@ffff9c4ec7a7df80 x1877759534471168/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772527.273414:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534471552/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772527.273494:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772527.273503:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772527.273508:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772527.273515:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534471168/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772527.273523:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1523 00000100:00080000:0.0:1790772527.273536:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772527.273554:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772527.273563:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790772527.273568:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772527.273594:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534471680/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772527.273606:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772527.273629:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772527.273630:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534471168:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772527.273727:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772527.273736:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772527.273738:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534471296:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772527.273755:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1523 00000100:00080000:0.0:1790772527.273769:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772527.273782:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772527.273796:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790772527.273810:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772527.273829:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534471808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772527.273851:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772527/real 1790772527] req@ffff9c4ec7a7f480 x1877759534471296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772527.273858:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772527.273875:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772527.273883:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772527.273899:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1523 00000100:00080000:0.0:1790772527.273909:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772527.273918:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534471296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772527.273927:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772527.273945:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772527.273973:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790772527.273977:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534471936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772527.273996:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772527.274014:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772527.274016:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534471296:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772527.274031:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772527.274040:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772527.274046:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534471424:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772527.274055:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1523 00000100:00080000:0.0:1790772527.274064:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772527.274076:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772527.274092:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772527.274122:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534472064/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772527.274150:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772527.274169:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772527.274177:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772527.274182:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772527.274191:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772527/real 1790772527] req@ffff9c4ec7a7c380 x1877759534471424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772527.274200:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1523 00000100:00080000:0.0:1790772527.274214:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772527.274231:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772527.274242:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772527.274244:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534471424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772527.274296:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790772527.274299:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534472192/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772527.274317:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772527.274337:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534471424:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772527.274357:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534471552:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772527.274410:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772527.274442:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772527/real 1790772527] req@ffff9c4ec7a7d880 x1877759534471552/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772527.274469:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534471552/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772527.274491:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772527.274499:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772527.274507:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534471552:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772527.274521:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534471680:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772527.274570:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772527.274595:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772527/real 1790772527] req@ffff9c4ec7a7ed80 x1877759534471680/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772527.274620:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534471680/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772527.274641:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772527.274687:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772527.274696:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534471680:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772527.274744:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534471808:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772527.274811:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772527.274838:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772527/real 1790772527] req@ffff9c4ec7a7ca80 x1877759534471808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772527.274867:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534471808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772527.274899:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772527.274908:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772527.274916:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534471808:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772527.274930:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534471936:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772527.274973:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772527.274998:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772527/real 1790772527] req@ffff9c4ec7a7e300 x1877759534471936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772527.275041:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534471936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772527.275060:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772527.275067:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772527.275074:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534471936:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772527.275087:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534472064:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772527.275129:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772527.275155:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772527/real 1790772527] req@ffff9c4ec7a7fb80 x1877759534472064/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772527.275182:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534472064/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772527.275205:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772527.275212:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772527.275220:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534472064:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772527.275232:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534472192:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772527.275300:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772527.275328:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772527/real 1790772527] req@ffff9c4ec7a7c700 x1877759534472192/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772527.275356:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534472192/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772527.275389:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772527.275397:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772527.275405:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534472192:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772527.336780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772527.336791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772527.336791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772527.336793:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772527.400796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772527.400857:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772527.400910:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772527.528784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772527.528791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772527.592754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772527.656791:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772527.720764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772527.720765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772527.720795:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772527.784786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772527.848769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772527.848794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772527.848810:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772527.976853:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772527.976858:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772527.976859:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772527.976863:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772528.040776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772528.040782:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772528.040786:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772528.040823:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772528.040859:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772528.040880:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772528.168758:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772528.232747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772528.296763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772528.360778:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772528.360818:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772528.360867:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772528.360874:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772528.424859:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772528.424865:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772528.424928:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772528.552793:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772528.552793:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772528.616744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772528.680738:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772528.744747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772528.744785:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772528.744802:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772528.808734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772528.872739:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772528.872744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772528.872746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772529.000761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772529.000778:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772529.000778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772529.000778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772529.064813:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772529.064820:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772529.064843:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772529.064857:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772529.064896:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772529.064905:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772529.192778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772529.256755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772529.320741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772529.384795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772529.384808:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772529.384827:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772529.384828:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772529.448782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772529.448909:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772529.448934:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772529.576756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772529.576789:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772529.640794:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772529.704778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772529.768794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772529.768840:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772529.768848:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772529.832769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772529.896766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772529.896792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772529.896809:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772530.024784:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772530.024791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772530.024820:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772530.024833:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772530.088754:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772530.088784:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772530.088786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772530.088863:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772530.088909:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772530.088922:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772530.216798:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772530.280748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772530.344751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772530.408827:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772530.408829:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772530.408840:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772530.408840:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772530.472801:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772530.472839:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772530.472988:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772530.600754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772530.600787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772530.664758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772530.728822:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772530.792752:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772530.792761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772530.792789:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772530.856788:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772530.920754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772530.920775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772530.920791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772531.048758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772531.048787:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772531.048795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772531.048800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772531.112801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772531.112827:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772531.112835:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772531.112868:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772531.112906:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772531.112941:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772531.240776:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772531.304772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772531.368739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772531.432784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772531.432792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772531.432801:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772531.432832:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772531.496824:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772531.496848:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772531.496958:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772531.624763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772531.624775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772531.688764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772531.752808:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772531.816795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772531.816829:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772531.816838:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772531.880777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772531.944817:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772531.944828:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772531.944935:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772532.072758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772532.072777:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772532.072797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772532.072811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772532.136792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772532.136819:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772532.136855:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772532.136858:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772532.136866:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772532.136868:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772532.264721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772532.328734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772532.392788:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772532.392798:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772532.392800:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772532.392807:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772532.392824:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1528 00000100:00080000:0.0:1790772532.392838:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772532.392854:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772532.392907:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534472320/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772532.392948:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772532.392953:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772532.392956:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772532.392962:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1528 00000100:00080000:0.0:1790772532.392967:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772532.392975:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772532.392989:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772532.393023:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534472448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772532.393036:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534472320:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772532.393043:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772532.393055:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772532.393057:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772532.393061:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1528 00000100:00080000:0.0:1790772532.393064:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772532.393067:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772532.393090:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772532.393110:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534472576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772532.393123:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772532.393128:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772532.393132:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772532.393139:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1528 00000100:00080000:0.0:1790772532.393147:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772532.393153:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772532.393162:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:2.0:1790772532.393225:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772532.393227:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534472704/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772532.393241:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772532.393244:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772532.393246:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772532.393251:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1528 00000100:00080000:0.0:1790772532.393255:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772532.393264:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772532.393272:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772532.393284:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534472832/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772532.393293:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772532.393296:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772532.393297:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772532.393303:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1528 00000100:00080000:0.0:1790772532.393307:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772532.393317:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772532/real 1790772532] req@ffff9c4ec7a7d180 x1877759534472320/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772532.393318:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772532.393330:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772532.393359:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534472960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772532.393371:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534472320/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772532.393377:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772532.393407:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772532.393410:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772532.393415:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772532.393416:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1528 00000100:00080000:0.0:1790772532.393430:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772532.393431:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772532.393444:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772532.393447:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534472320:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790772532.393456:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772532.393472:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534473088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772532.393480:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772532.393482:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772532.393484:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772532.393489:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1528 00000100:00100000:2.0:1790772532.393495:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534472448:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772532.393497:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772532.393511:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772532.393517:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772532.393534:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534473216/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772532.393545:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772532.393551:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772532.393553:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772532.393558:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772532.393566:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1528 00000100:00080000:0.0:1790772532.393570:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772532.393581:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772532/real 1790772532] req@ffff9c4ec7a7c700 x1877759534472448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772532.393582:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772532.393595:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772532.393629:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534472448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772532.393631:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534473344/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772532.393705:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772532.393713:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772532.393725:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534472448:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772532.393758:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534472576:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772532.393804:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772532.393825:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772532/real 1790772532] req@ffff9c4ec7a7fb80 x1877759534472576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772532.393842:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534472576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772532.393854:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772532.393859:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772532.393863:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534472576:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772532.393872:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534472704:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772532.393894:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772532.393909:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772532/real 1790772532] req@ffff9c4ec7a7e300 x1877759534472704/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772532.393924:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534472704/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772532.393937:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772532.393941:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772532.393945:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534472704:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772532.393953:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534472832:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772532.393973:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772532.393997:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772532/real 1790772532] req@ffff9c4ec7a7ca80 x1877759534472832/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772532.394011:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534472832/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772532.394023:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772532.394027:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772532.394032:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534472832:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772532.394039:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534472960:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772532.394058:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772532.394076:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772532/real 1790772532] req@ffff9c4ec7a7ed80 x1877759534472960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772532.394091:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534472960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772532.394113:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772532.394118:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772532.394123:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534472960:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772532.394131:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534473088:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772532.394152:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772532.394165:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772532/real 1790772532] req@ffff9c4ec7a7d880 x1877759534473088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772532.394223:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534473088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772532.394235:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772532.394239:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772532.394244:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534473088:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772532.394276:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534473216:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772532.394302:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772532.394318:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772532/real 1790772532] req@ffff9c4ec7a7c380 x1877759534473216/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772532.394333:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534473216/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772532.394350:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772532.394357:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772532.394367:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534473216:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772532.394381:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534473344:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772532.394421:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772532.394445:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772532/real 1790772532] req@ffff9c4ec7a7f480 x1877759534473344/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772532.394470:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534473344/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772532.394491:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772532.394498:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772532.394506:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534473344:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772532.456761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772532.456766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772532.456777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772532.456845:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772532.520773:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772532.520901:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772532.520965:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772532.648759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772532.648804:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772532.712756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772532.776831:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772532.840765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772532.840772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772532.840847:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772532.904743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772532.968761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772532.968761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772532.968799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772533.096768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772533.096785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772533.096785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772533.096785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772533.160767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772533.160777:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772533.160794:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772533.160823:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772533.160855:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772533.160873:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772533.288760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772533.352750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772533.416762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772533.480756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772533.480761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772533.480777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772533.480777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772533.544759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772533.544854:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772533.544866:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772533.672766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772533.672782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772533.736777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772533.800748:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772533.864770:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772533.864774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772533.864780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772533.928789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772533.992722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772533.992751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772533.992751:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772534.120771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772534.120844:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772534.120849:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772534.120850:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772534.184758:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772534.184797:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772534.184816:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772534.184821:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772534.184834:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772534.184875:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772534.312786:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772534.376777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772534.440747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772534.504818:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772534.504872:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772534.504873:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772534.504874:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772534.568884:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772534.568898:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772534.568985:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772534.696782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772534.696826:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772534.760748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772534.824848:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772534.888795:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772534.888815:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772534.888815:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772534.952748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772535.016814:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772535.016816:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772535.016853:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772535.144864:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772535.144873:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772535.144889:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772535.144894:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772535.208845:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772535.208845:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772535.208871:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772535.208872:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772535.208919:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772535.208938:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772535.336784:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772535.400771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772535.464777:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772535.528785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772535.528800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772535.528897:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772535.528902:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772535.592769:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772535.592853:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772535.592912:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772535.720776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772535.720811:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772535.784775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772535.848757:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772535.912763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772535.912784:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772535.912806:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772535.976851:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772536.040762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772536.040828:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772536.040829:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772536.168790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772536.168834:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772536.168835:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772536.168836:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772536.232774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772536.232806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772536.232836:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772536.232842:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772536.232860:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772536.232894:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772536.360749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772536.424780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772536.488775:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772536.552772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772536.552773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772536.552779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772536.552779:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772536.616735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772536.616800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772536.616853:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772536.744708:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772536.744722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772536.808706:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772536.872725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772536.936727:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772536.936727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772536.936736:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772537.000713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772537.064716:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772537.064745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772537.064779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772537.192737:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772537.192737:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772537.192737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772537.192737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772537.256718:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772537.256734:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772537.256738:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772537.256739:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772537.256767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772537.256768:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772537.384714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772537.448743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772537.512775:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772537.512788:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772537.512795:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772537.512801:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772537.512813:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1533 00000100:00080000:0.0:1790772537.512849:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772537.512870:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772537.512938:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534473472/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772537.512999:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772537.513004:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772537.513007:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772537.513013:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1533 00000100:00080000:0.0:1790772537.513019:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772537.513026:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772537.513042:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772537.513054:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534473472:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772537.513067:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534473600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772537.513125:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772537.513130:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772537.513132:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772537.513138:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1533 00000100:00080000:0.0:1790772537.513144:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772537.513150:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772537.513157:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772537.513192:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534473728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772537.513206:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772537.513211:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772537.513214:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772537.513233:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1533 00000800:00000100:2.0:1790772537.513235:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772537.513246:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772537.513283:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772537.513291:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772537.513309:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534473856/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772537.513322:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772537.513327:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772537.513330:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772537.513354:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1533 00000100:00080000:0.0:1790772537.513361:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772537.513363:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772537/real 1790772537] req@ffff9c4ec7a7df80 x1877759534473472/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772537.513375:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772537.513388:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772537.513423:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534473984/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772537.513427:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534473472/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772537.513455:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772537.513461:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772537.513462:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772537.513465:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772537.513472:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772537.513474:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1533 00000100:00080000:0.0:1790772537.513482:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772537.513483:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534473472:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772537.513493:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772537.513501:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772537.513525:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534474112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772537.513525:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534473600:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772537.513556:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772537.513564:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:2.0:1790772537.513570:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772537.513571:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772537.513585:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1533 00000100:00080000:0.0:1790772537.513592:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772537.513593:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772537/real 1790772537] req@ffff9c4ec7a7f480 x1877759534473600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772537.513604:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772537.513619:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772537.513623:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534473600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772537.513642:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534474240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772537.513649:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772537.513656:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772537.513661:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772537.513663:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772537.513669:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772537.513672:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534473600:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772537.513679:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1533 00000100:00080000:0.0:1790772537.513723:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772537.513727:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534473728:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772537.513733:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772537.513769:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772537.513790:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534474368/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772537.513813:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772537.513817:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772537.513823:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772537.513826:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772537.513834:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1533 00000100:00080000:0.0:1790772537.513840:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772537.513845:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772537/real 1790772537] req@ffff9c4ec7a7c380 x1877759534473728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772537.513853:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772537.513865:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772537.513887:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534474496/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772537.513893:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534473728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772537.513930:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772537.513935:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772537.513940:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534473728:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772537.513951:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534473856:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772537.513977:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772537.513988:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772537/real 1790772537] req@ffff9c4ec7a7d880 x1877759534473856/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772537.514002:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534473856/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772537.514014:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772537.514018:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772537.514022:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534473856:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772537.514029:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534473984:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772537.514046:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772537.514056:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772537/real 1790772537] req@ffff9c4ec7a7ed80 x1877759534473984/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772537.514069:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534473984/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772537.514135:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772537.514139:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772537.514143:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534473984:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772537.514185:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534474112:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772537.514212:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772537.514225:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772537/real 1790772537] req@ffff9c4ec7a7ca80 x1877759534474112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772537.514238:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534474112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772537.514258:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772537.514262:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772537.514266:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534474112:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772537.514273:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534474240:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772537.514290:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772537.514301:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772537/real 1790772537] req@ffff9c4ec7a7e300 x1877759534474240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772537.514314:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534474240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772537.514323:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772537.514326:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772537.514331:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534474240:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772537.514337:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534474368:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772537.514353:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772537.514363:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772537/real 1790772537] req@ffff9c4ec7a7fb80 x1877759534474368/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772537.514375:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534474368/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772537.514387:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772537.514391:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772537.514395:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534474368:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772537.514418:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534474496:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772537.514436:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772537.514445:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772537/real 1790772537] req@ffff9c4ec7a7c700 x1877759534474496/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772537.514458:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534474496/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772537.514468:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772537.514471:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772537.514475:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534474496:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772537.576790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772537.576853:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772537.576854:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772537.576867:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772537.640839:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772537.640878:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772537.640958:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772537.768766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772537.768813:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772537.832794:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772537.896759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772537.960773:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772537.960816:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772537.960834:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772538.024743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772538.088785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772538.088786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772538.088850:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772538.216759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772538.216764:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772538.216766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772538.216767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772538.280795:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772538.280850:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772538.280853:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772538.280855:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772538.280916:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772538.280935:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772538.408746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772538.472782:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772538.536750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772538.600814:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772538.600829:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772538.600831:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772538.600842:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772538.664830:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772538.664837:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772538.664923:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772538.792756:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772538.792804:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772538.856783:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772538.920761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772538.984793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772538.984793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772538.984797:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772539.048758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772539.112774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772539.112787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772539.112799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772539.240793:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772539.240806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772539.240813:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772539.240844:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772539.304742:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772539.304827:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772539.304844:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772539.304867:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772539.304869:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772539.304909:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772539.432777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772539.496771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772539.560796:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772539.624749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772539.624770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772539.624779:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772539.624836:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772539.688791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772539.688834:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772539.688993:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772539.816815:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772539.816826:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772539.880779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772539.944869:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772540.008875:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772540.008878:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772540.008879:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772540.072786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772540.136778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772540.136784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772540.136842:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772540.264858:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772540.264862:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772540.264903:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772540.264910:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772540.328800:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772540.328806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772540.328812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772540.328816:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772540.328868:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772540.328892:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772540.456762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772540.520762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772540.584844:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772540.648772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772540.648833:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772540.648874:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772540.648877:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772540.712872:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772540.712885:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772540.713116:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772540.840814:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772540.840814:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772540.904742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772540.968810:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772541.032784:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772541.032792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772541.033704:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772541.096760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772541.160770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772541.160792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772541.160802:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772541.288763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772541.288771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772541.288785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772541.288823:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772541.352779:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772541.352805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772541.352814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772541.352859:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772541.352864:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772541.352870:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772541.480719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772541.544697:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772541.608731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772541.672739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772541.672773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772541.672793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772541.672800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772541.736718:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772541.736766:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772541.736801:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772541.864782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772541.864800:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772541.928748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772541.992722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772542.056720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772542.056720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772542.056737:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772542.120728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772542.184725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772542.184725:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772542.184777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772542.312716:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772542.312718:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772542.312722:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772542.312743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772542.376721:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772542.376727:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772542.376736:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772542.376755:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772542.376771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772542.376789:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772542.504716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772542.568710:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772542.632725:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772542.632736:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772542.632740:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772542.632755:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1538 00000100:00100000:2.0:1790772542.632755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772542.632769:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772542.632784:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772542.632829:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534474624/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772542.632855:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772542.632858:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772542.632860:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772542.632864:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1538 00000100:00080000:0.0:1790772542.632867:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772542.632872:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772542.632884:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772542.632895:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534474624:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772542.632901:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534474752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772542.632913:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772542.632916:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772542.632918:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772542.632921:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1538 00000100:00080000:0.0:1790772542.632924:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772542.632927:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772542.632931:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772542.632940:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534474880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772542.632948:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772542.632952:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772542.632954:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772542.632958:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1538 00000100:00080000:0.0:1790772542.632962:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772542.632968:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772542.632984:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772542.632996:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534475008/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772542.633003:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772542.633017:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772542.633020:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772542.633033:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772542.633040:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1538 00000100:00080000:0.0:1790772542.633044:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772542.633050:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772542.633054:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772542.633064:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534475136/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772542.633074:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772542.633078:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772542.633081:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772542.633086:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1538 00000100:00080000:0.0:1790772542.633090:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772542.633097:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772542.633099:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772542/real 1790772542] req@ffff9c4ec7a7d180 x1877759534474624/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790772542.633103:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772542.633138:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534475264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772542.633159:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772542.633163:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772542.633166:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772542.633171:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534474624/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772542.633176:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1538 00000100:00080000:0.0:1790772542.633193:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772542.633199:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772542.633212:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772542.633226:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534475392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772542.633232:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772542.633235:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772542.633238:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772542.633241:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772542.633243:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772542.633250:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1538 00000100:00100000:2.0:1790772542.633252:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534474624:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772542.633258:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772542.633265:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772542.633269:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772542.633276:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534475520/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772542.633283:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534474752:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772542.633285:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772542.633295:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772542.633297:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772542.633302:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1538 00000100:00080000:0.0:1790772542.633307:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772542.633311:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772542.633315:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:2.0:1790772542.633320:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772542.633337:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534475648/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772542.633345:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772542/real 1790772542] req@ffff9c4ec7a7c700 x1877759534474752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772542.633380:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534474752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772542.633397:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772542.633400:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772542.633403:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534474752:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772542.633408:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534474880:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772542.633422:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772542.633429:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772542/real 1790772542] req@ffff9c4ec7a7fb80 x1877759534474880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772542.633437:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534474880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772542.633444:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772542.633446:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772542.633448:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534474880:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772542.633453:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534475008:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772542.633464:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772542.633471:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772542/real 1790772542] req@ffff9c4ec7a7e300 x1877759534475008/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772542.633479:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534475008/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772542.633485:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772542.633487:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772542.633490:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534475008:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772542.633495:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534475136:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772542.633505:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772542.633511:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772542/real 1790772542] req@ffff9c4ec7a7ca80 x1877759534475136/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772542.633519:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534475136/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772542.633533:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772542.633535:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772542.633537:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534475136:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772542.633542:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534475264:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772542.633554:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772542.633562:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772542/real 1790772542] req@ffff9c4ec7a7ed80 x1877759534475264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772542.633570:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534475264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772542.633581:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772542.633584:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772542.633586:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534475264:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772542.633591:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534475392:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772542.633601:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772542.633608:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772542/real 1790772542] req@ffff9c4ec7a7d880 x1877759534475392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772542.633644:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534475392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772542.633661:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772542.633730:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772542.633737:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534475392:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772542.633785:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534475520:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772542.633803:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772542.633812:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772542/real 1790772542] req@ffff9c4ec7a7c380 x1877759534475520/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772542.633820:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534475520/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772542.633845:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772542.633849:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772542.633852:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534475520:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772542.633859:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534475648:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772542.633876:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772542.633886:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772542/real 1790772542] req@ffff9c4ec7a7f480 x1877759534475648/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772542.633897:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534475648/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772542.633907:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772542.633910:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772542.633914:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534475648:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772542.696736:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772542.696737:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772542.696741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772542.696754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772542.760742:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772542.760845:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772542.761021:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772542.888745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772542.888752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772542.952735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772543.016748:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772543.080705:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772543.080706:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772543.080736:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772543.144757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772543.208809:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772543.208818:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772543.208819:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772543.336764:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772543.336774:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772543.336828:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772543.336837:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772543.400761:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772543.400793:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772543.400794:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772543.400799:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772543.400837:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772543.400850:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772543.528790:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772543.592778:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772543.656806:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772543.720778:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772543.720797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772543.720815:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772543.720830:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772543.784816:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772543.784852:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772543.784899:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772543.912805:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772543.912818:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772543.976790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772544.040753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772544.104753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772544.104775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772544.104808:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772544.168789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772544.232771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772544.232820:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772544.232876:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772544.360788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772544.360789:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772544.360806:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772544.360810:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772544.424788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772544.424796:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772544.424812:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772544.424824:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772544.424856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772544.424871:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772544.552789:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772544.616751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772544.680865:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772544.744857:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772544.744874:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772544.744887:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772544.744905:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772544.808820:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772544.808848:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772544.809024:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772544.936757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772544.936844:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772545.000769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772545.064797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772545.128733:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772545.128733:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772545.128771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772545.192781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772545.256750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772545.256770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772545.256799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772545.384760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772545.384784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772545.384798:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772545.384806:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772545.448792:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772545.448819:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772545.448824:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772545.448880:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772545.448880:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772545.448953:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772545.576703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772545.640754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772545.704759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772545.768812:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772545.768812:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772545.768848:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772545.768867:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772545.832778:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772545.832833:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772545.832947:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772545.960741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772545.960766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772546.024738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772546.088842:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772546.152756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772546.152759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772546.152768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772546.216805:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772546.280792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772546.280807:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772546.280874:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772546.408766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772546.408811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772546.408830:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772546.408846:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772546.472760:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772546.472760:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772546.472777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772546.472777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772546.472817:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772546.472843:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772546.600728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772546.664737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772546.728773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772546.792800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772546.792830:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772546.792832:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772546.792833:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772546.856799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772546.856802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772546.856864:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772546.984792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772546.984804:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772547.048746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772547.112850:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772547.176728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772547.176734:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772547.176734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772547.240724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772547.304826:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772547.304836:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772547.304838:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772547.432778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772547.432797:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772547.432806:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772547.432813:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772547.496798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772547.496802:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772547.496807:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772547.496828:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772547.496862:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772547.496863:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772547.624771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772547.688735:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772547.752785:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772547.752823:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772547.752833:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772547.752841:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772547.752860:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1543 00000100:00080000:3.0:1790772547.752915:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772547.752938:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772547.753002:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534475776/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772547.753057:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772547.753063:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772547.753066:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772547.753072:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1543 00000100:00080000:3.0:1790772547.753077:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772547.753085:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772547.753101:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772547.753127:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534475904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772547.753129:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534475776:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772547.753159:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772547.753164:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772547.753166:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772547.753173:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1543 00000100:00080000:3.0:1790772547.753178:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772547.753184:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772547.753192:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772547.753210:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534476032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772547.753223:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772547.753229:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772547.753232:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772547.753237:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1543 00000100:00080000:3.0:1790772547.753245:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772547.753251:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772547.753257:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:2.0:1790772547.753271:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772547.753277:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534476160/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772547.753292:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772547.753297:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772547.753300:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772547.753306:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1543 00000100:00080000:3.0:1790772547.753311:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772547.753320:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772547.753326:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772547.753340:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534476288/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772547.753353:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772547.753358:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772547.753362:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772547.753373:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772547/real 1790772547] req@ffff9c4ef73c0e00 x1877759534475776/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772547.753373:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1543 00000100:00080000:3.0:1790772547.753383:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772547.753395:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772547.753407:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772547.753430:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534476416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772547.753445:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772547.753447:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534475776/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772547.753450:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772547.753458:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772547.753479:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1543 00000100:00080000:2.0:1790772547.753488:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772547.753492:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772547.753511:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772547.753516:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772547.753531:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534475776:192.168.204.150@tcp:250:kworker.0 00000008:00100000:3.0:1790772547.753534:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772547.753569:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534476544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772547.753593:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772547.753599:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772547.753602:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772547.753611:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534475904:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772547.753613:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1543 00000100:00080000:3.0:1790772547.753625:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772547.753636:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772547.753644:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772547.753714:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534476672/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772547.753733:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772547.753734:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772547.753746:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772547.753750:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772547.753760:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1543 00000100:00080000:3.0:1790772547.753767:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772547.753775:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772547.753783:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772547/real 1790772547] req@ffff9c4ef73c0000 x1877759534475904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772547.753787:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772547.753811:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534476800/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772547.753819:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534475904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772547.753868:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772547.753901:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772547.753919:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534475904:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772547.753932:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534476032:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772547.753968:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772547.753985:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772547/real 1790772547] req@ffff9c4ef73c1c00 x1877759534476032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772547.754002:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534476032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772547.754014:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772547.754019:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772547.754034:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534476032:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772547.754042:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534476160:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772547.754064:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772547.754076:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772547/real 1790772547] req@ffff9c4ef73c2a00 x1877759534476160/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772547.754092:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534476160/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772547.754105:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772547.754109:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772547.754113:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534476160:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772547.754121:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534476288:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772547.754152:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772547.754164:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772547/real 1790772547] req@ffff9c4ef73c0700 x1877759534476288/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772547.754178:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534476288/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772547.754190:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772547.754194:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772547.754199:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534476288:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772547.754206:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534476416:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772547.754225:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772547.754237:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772547/real 1790772547] req@ffff9c4ef73c2300 x1877759534476416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772547.754261:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534476416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772547.754280:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772547.754284:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772547.754289:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534476416:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772547.754296:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534476544:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772547.754315:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772547.754328:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772547/real 1790772547] req@ffff9c4ef73c1500 x1877759534476544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772547.754343:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534476544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772547.754354:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772547.754358:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772547.754362:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534476544:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772547.754395:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534476672:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772547.754417:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772547.754429:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772547/real 1790772547] req@ffff9c4ef73c0a80 x1877759534476672/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772547.754444:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534476672/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772547.754466:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772547.754470:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772547.754475:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534476672:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772547.754482:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534476800:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772547.754500:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772547.754512:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772547/real 1790772547] req@ffff9c4ef73c3800 x1877759534476800/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772547.754526:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534476800/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772547.754538:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772547.754542:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772547.754546:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534476800:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772547.816825:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772547.816825:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772547.816835:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772547.816838:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772547.880816:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772547.880817:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772547.880915:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772548.008768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772548.008805:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772548.072769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772548.136711:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772548.200797:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772548.200797:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772548.200798:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772548.264854:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772548.328790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772548.328799:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772548.328801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772548.456751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772548.456825:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772548.456942:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772548.456984:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772548.520743:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772548.520745:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772548.520756:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772548.520781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772548.520826:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772548.520832:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772548.648712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772548.712718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772548.776708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772548.840783:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772548.840785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772548.840798:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772548.840901:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772548.904730:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772548.904760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772548.904909:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772549.032794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772549.032803:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772549.096757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772549.160800:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772549.224765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772549.224767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772549.224791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772549.288739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772549.352767:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772549.352781:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772549.352788:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772549.480865:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772549.480920:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772549.481052:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772549.481171:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772549.544764:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772549.544773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772549.544811:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772549.544813:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772549.544858:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772549.544892:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772549.672750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772549.736795:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772549.800728:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772549.864718:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772549.864718:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772549.864726:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772549.864746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772549.928769:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772549.928775:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772549.928895:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772550.056724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772550.056735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772550.120765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772550.184753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772550.248708:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772550.248749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772550.248813:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772550.312766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772550.376744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772550.376750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772550.376762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772550.504807:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772550.504807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772550.504847:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772550.504848:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772550.568715:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772550.568717:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772550.568719:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772550.568727:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772550.568770:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772550.568776:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772550.696742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772550.760788:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772550.824737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772550.888770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772550.888785:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772550.888786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772550.888789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772550.952756:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772550.952790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772550.952860:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772551.080747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772551.080779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772551.144760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772551.208799:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772551.272848:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772551.272872:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772551.272874:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772551.336780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772551.400742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772551.400747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772551.400783:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772551.528844:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772551.528857:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772551.528941:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772551.529049:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772551.592785:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772551.592881:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772551.592885:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772551.592889:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772551.592896:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772551.592945:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772551.720788:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772551.784776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772551.848757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772551.912735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772551.912775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772551.912848:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772551.912848:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772551.976823:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772551.976829:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772551.976837:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772552.104772:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772552.104783:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772552.168738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772552.232820:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772552.296770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772552.296771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772552.296806:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772552.360765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772552.424772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772552.424812:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772552.424824:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772552.552930:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772552.552986:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772552.553734:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772552.553842:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772552.616809:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772552.616834:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772552.616859:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772552.616887:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772552.616918:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772552.616920:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772552.744803:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772552.808802:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772552.872754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772552.872854:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772552.872864:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772552.872871:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772552.872887:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1548 00000100:00080000:1.0:1790772552.872905:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772552.872925:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772552.873010:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534476928/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772552.873052:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772552.873058:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772552.873060:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772552.873066:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1548 00000100:00080000:1.0:1790772552.873072:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772552.873089:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772552.873107:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772552.873114:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534476928:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772552.873133:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534477056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772552.873149:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772552.873155:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772552.873158:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772552.873165:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1548 00000100:00080000:1.0:1790772552.873169:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772552.873176:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772552.873183:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772552.873199:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534477184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772552.873213:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772552.873218:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772552.873222:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772552.873227:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1548 00000100:00080000:1.0:1790772552.873234:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772552.873245:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772552.873246:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772552.873255:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772552.873272:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534477312/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772552.873286:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772552.873291:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772552.873295:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772552.873301:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1548 00000100:00080000:1.0:1790772552.873306:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772552.873311:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772552.873318:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772552.873336:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534477440/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772552.873345:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772552/real 1790772552] req@ffff9c4efac4fb80 x1877759534476928/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772552.873360:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772552.873373:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772552.873379:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772552.873389:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1548 00000100:00080000:1.0:1790772552.873397:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772552.873401:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534476928/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772552.873413:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772552.873427:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790772552.873429:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772552.873437:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772552.873448:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534476928:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772552.873466:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534477568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772552.873492:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534477056:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772552.873494:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772552.873509:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772552.873515:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772552.873528:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1548 00000100:00080000:1.0:1790772552.873536:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772552.873544:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772552.873553:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772552.873563:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772552.873580:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772552/real 1790772552] req@ffff9c4efac4df80 x1877759534477056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772552.873588:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534477696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772552.873642:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772552.873695:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534477056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772552.873700:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772552.873708:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772552.873723:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1548 00000100:00080000:1.0:1790772552.873731:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772552.873740:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772552.873741:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790772552.873750:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:2.0:1790772552.873753:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772552.873764:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534477056:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772552.873803:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534477824/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772552.873808:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534477184:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772552.873836:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772552.873842:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772552.873846:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772552.873855:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1548 00000100:00080000:1.0:1790772552.873862:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772552.873867:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772552.873876:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772552.873887:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772552.873896:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772552/real 1790772552] req@ffff9c4efac4e300 x1877759534477184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772552.873912:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534477952/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772552.873922:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534477184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772552.873952:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772552.873959:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772552.873965:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534477184:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772552.873976:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534477312:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772552.874002:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772552.874015:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772552/real 1790772552] req@ffff9c4efac4f480 x1877759534477312/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772552.874031:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534477312/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772552.874045:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772552.874049:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772552.874053:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534477312:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772552.874061:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534477440:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772552.874082:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772552.874094:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772552/real 1790772552] req@ffff9c4efac4f800 x1877759534477440/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772552.874111:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534477440/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772552.874123:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772552.874128:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772552.874132:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534477440:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772552.874158:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534477568:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772552.874181:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772552.874193:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772552/real 1790772552] req@ffff9c4efac4dc00 x1877759534477568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772552.874218:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534477568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772552.874248:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772552.874256:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772552.874264:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534477568:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772552.874277:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534477696:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772552.874317:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772552.874335:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772552/real 1790772552] req@ffff9c4efac4ed80 x1877759534477696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772552.874351:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534477696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772552.874373:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772552.874377:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772552.874382:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534477696:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772552.874390:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534477824:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772552.874412:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772552.874425:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772552/real 1790772552] req@ffff9c4efac4c700 x1877759534477824/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772552.874440:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534477824/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772552.874454:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772552.874458:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772552.874462:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534477824:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772552.874470:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534477952:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772552.874490:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772552.874502:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772552/real 1790772552] req@ffff9c4efac4ea00 x1877759534477952/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772552.874517:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534477952/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772552.874529:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772552.874533:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772552.874539:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534477952:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772552.936764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772552.936800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772552.936827:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772552.936832:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772553.000829:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772553.000867:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772553.000922:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772553.128749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772553.128833:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772553.192835:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772553.256755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772553.320755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772553.320802:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772553.320846:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772553.384752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772553.448739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772553.448787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772553.448810:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772553.576830:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772553.576877:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772553.576993:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772553.577029:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772553.640828:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772553.640835:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772553.640890:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772553.640898:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772553.640936:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772553.640940:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772553.768810:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772553.832788:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772553.896816:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772553.960767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772553.960767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772553.960778:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772553.960781:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772554.024765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772554.024779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772554.024832:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772554.152797:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772554.152812:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772554.216812:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772554.280956:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772554.344784:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772554.344797:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772554.344808:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772554.408767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772554.472784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772554.472812:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772554.472853:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772554.600800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772554.600810:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772554.600820:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772554.600850:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772554.664811:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772554.664816:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772554.664816:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772554.664816:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772554.664866:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772554.664876:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772554.792777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772554.856721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772554.920708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772554.984747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772554.984761:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772554.984762:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772554.984762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772555.048723:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772555.048810:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772555.048862:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772555.176762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772555.176842:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772555.240830:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772555.304818:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772555.368774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772555.368815:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772555.368844:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772555.432796:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772555.496796:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772555.496803:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772555.496807:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772555.624779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772555.624810:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772555.624815:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772555.624821:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772555.688797:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772555.688813:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772555.688825:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772555.688860:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772555.688895:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772555.688897:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772555.816816:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772555.880820:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772555.944774:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772556.008782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772556.008802:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772556.008820:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772556.008842:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772556.072846:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772556.072861:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772556.072881:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772556.200768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772556.200824:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772556.264791:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772556.328812:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772556.392785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772556.392810:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772556.392829:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772556.456787:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772556.520780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772556.520806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772556.520825:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772556.648806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772556.648811:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772556.648817:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772556.648887:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772556.712822:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772556.712836:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772556.712836:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772556.712856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772556.712907:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772556.712945:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772556.840768:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772556.904797:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772556.968773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772557.032823:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772557.032832:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772557.032833:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772557.032839:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772557.096817:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772557.096865:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772557.096916:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772557.224756:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772557.224800:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772557.288754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772557.352766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772557.416777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772557.416809:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772557.416822:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772557.480755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772557.544793:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772557.544798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772557.544817:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772557.672824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772557.672852:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772557.672896:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772557.672927:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772557.736801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772557.736808:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772557.736816:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772557.736869:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772557.736873:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772557.736928:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772557.864794:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772557.928795:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772557.992765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772557.992843:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772557.992858:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772557.992869:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772557.992891:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1553 00000100:00080000:1.0:1790772557.992914:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772557.992938:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772557.993012:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534478080/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772557.993070:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772557.993078:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772557.993083:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772557.993093:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1553 00000100:00080000:1.0:1790772557.993103:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772557.993116:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772557.993138:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772557.993145:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534478080:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772557.993175:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534478208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772557.993203:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772557.993212:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772557.993217:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772557.993227:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1553 00000100:00080000:1.0:1790772557.993236:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772557.993248:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772557.993260:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790772557.993284:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772557.993290:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534478336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772557.993315:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772557.993324:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772557.993330:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772557.993339:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1553 00000100:00080000:1.0:1790772557.993347:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772557.993361:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772557.993373:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772557.993377:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772557/real 1790772557] req@ffff9c4efac4e680 x1877759534478080/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772557.993404:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534478464/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772557.993422:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534478080/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772557.993436:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772557.993454:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772557.993459:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772557.993461:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772557.993476:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1553 00000100:00080000:2.0:1790772557.993478:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772557.993491:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772557.993495:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534478080:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772557.993509:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772557.993521:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772557.993546:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534478592/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772557.993554:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534478208:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772557.993592:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772557.993602:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772557.993607:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790772557.993619:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772557.993621:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1553 00000100:00080000:1.0:1790772557.993637:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772557.993651:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772557.993729:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772557/real 1790772557] req@ffff9c4efac4ea00 x1877759534478208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772557.993755:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772557.993773:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534478208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772557.993792:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534478720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772557.993816:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772557.993817:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772557.993832:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772557.993837:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772557.993839:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772557.993850:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1553 00000100:00100000:2.0:1790772557.993858:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534478208:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772557.993861:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772557.993879:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772557.993884:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534478336:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790772557.993892:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772557.993926:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534478848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772557.993938:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772557.993961:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772557.993966:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772557/real 1790772557] req@ffff9c4efac4c700 x1877759534478336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772557.993975:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772557.993986:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772557.993998:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534478336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772557.994002:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1553 00000100:00080000:1.0:1790772557.994018:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772557.994023:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772557.994032:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772557.994038:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772557.994044:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534478336:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790772557.994057:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772557.994058:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534478464:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772557.994096:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534478976/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772557.994115:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772557.994133:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772557.994143:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772557.994153:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772557.994154:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772557/real 1790772557] req@ffff9c4efac4ed80 x1877759534478464/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772557.994170:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1553 00000100:00080000:1.0:1790772557.994184:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772557.994201:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772557.994206:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534478464/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772557.994215:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:2.0:1790772557.994228:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772557.994235:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772557.994242:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534478464:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772557.994252:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534479104/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772557.994279:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534478592:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772557.994316:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772557.994333:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772557/real 1790772557] req@ffff9c4efac4dc00 x1877759534478592/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772557.994349:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534478592/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772557.994362:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772557.994367:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772557.994372:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534478592:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772557.994379:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534478720:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772557.994400:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772557.994412:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772557/real 1790772557] req@ffff9c4efac4f800 x1877759534478720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772557.994427:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534478720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772557.994457:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772557.994462:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772557.994467:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534478720:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772557.994474:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534478848:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772557.994493:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772557.994505:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772557/real 1790772557] req@ffff9c4efac4f480 x1877759534478848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772557.994520:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534478848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772557.994531:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772557.994534:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772557.994539:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534478848:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772557.994546:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534478976:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772557.994566:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772557.994578:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772557/real 1790772557] req@ffff9c4efac4e300 x1877759534478976/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772557.994593:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534478976/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772557.994606:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772557.994610:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772557.994615:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534478976:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772557.994634:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534479104:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772557.994711:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772557.994730:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772557/real 1790772557] req@ffff9c4efac4df80 x1877759534479104/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772557.994747:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534479104/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772557.994760:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772557.994765:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772557.994770:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534479104:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772558.056787:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772558.056813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772558.056822:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772558.056833:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772558.120801:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772558.120813:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772558.120929:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772558.248781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772558.248790:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772558.312779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772558.376803:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772558.440780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772558.440813:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772558.440814:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772558.504797:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772558.568828:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772558.568828:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772558.568855:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772558.696829:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772558.696871:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772558.696876:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772558.696927:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772558.760766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772558.760787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772558.760841:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772558.760841:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772558.760860:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772558.760866:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772558.888793:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772558.952796:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772559.016762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772559.080780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772559.080800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772559.080807:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772559.080817:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772559.144855:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772559.144874:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772559.144911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772559.272790:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772559.272790:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772559.336769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772559.400794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772559.464751:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772559.464757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772559.464769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772559.528774:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772559.592788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772559.592799:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772559.592820:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772559.720834:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772559.720844:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772559.720872:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772559.720885:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772559.784768:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772559.784776:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772559.784824:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772559.784839:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772559.784864:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772559.784900:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772559.912788:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772559.976778:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772560.040785:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772560.104806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772560.104808:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772560.104821:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772560.104831:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772560.168806:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772560.168822:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772560.168941:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772560.296760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772560.296781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772560.360811:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772560.424782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772560.488776:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772560.488799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772560.488814:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772560.552764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772560.616820:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772560.616821:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772560.616825:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772560.744797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772560.744833:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772560.744851:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772560.744896:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772560.808789:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772560.808792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772560.808843:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772560.808851:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772560.808880:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772560.808885:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772560.936779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772561.000829:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772561.064791:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772561.128809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772561.128809:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772561.128813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772561.128820:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772561.192807:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772561.192840:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772561.192900:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772561.320761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772561.320785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772561.384784:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772561.448885:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772561.512740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772561.512783:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772561.512839:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772561.576778:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772561.640753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772561.640779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772561.640829:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772561.768790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772561.768792:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772561.768803:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772561.768864:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772561.832749:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772561.832782:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772561.832831:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772561.832859:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772561.832868:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772561.832879:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772561.960760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772562.024771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772562.088752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772562.152758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772562.152800:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772562.152828:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772562.152839:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772562.216771:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772562.216771:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772562.216937:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772562.344794:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772562.344817:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772562.408781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772562.472816:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772562.536753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772562.536758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772562.536787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772562.600806:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772562.664760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772562.664790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772562.664794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772562.792803:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772562.792804:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772562.792808:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772562.792809:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772562.856801:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772562.856801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772562.856802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772562.856846:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772562.856898:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772562.856910:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772562.984743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772563.048759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772563.112782:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772563.112830:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772563.112837:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772563.112844:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772563.112859:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1559 00000100:00080000:2.0:1790772563.112878:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772563.112897:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772563.112965:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534479232/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772563.113016:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772563.113023:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772563.113028:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772563.113036:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1559 00000100:00080000:2.0:1790772563.113045:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772563.113056:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772563.113076:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772563.113105:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534479360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772563.113129:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772563.113137:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1790772563.113142:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534479232:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772563.113143:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772563.113158:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1559 00000100:00080000:2.0:1790772563.113165:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772563.113174:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772563.113182:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772563.113205:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534479488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772563.113218:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772563.113224:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772563.113227:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772563.113232:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1559 00000100:00080000:2.0:1790772563.113257:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772563.113265:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772563.113272:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772563.113285:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772563.113291:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534479616/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772563.113307:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772563.113312:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772563.113315:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772563.113321:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1559 00000100:00080000:2.0:1790772563.113326:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772563.113332:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772563.113338:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772563.113352:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534479744/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772563.113367:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772563/real 1790772563] req@ffff9c4efbcc2680 x1877759534479232/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772563.113368:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772563.113383:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772563.113390:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772563.113403:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1559 00000100:00080000:2.0:1790772563.113409:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772563.113416:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772563.113424:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534479232/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790772563.113427:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772563.113451:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534479872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772563.113452:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772563.113469:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772563.113483:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772563.113485:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534479232:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772563.113490:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772563.113497:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772563.113507:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1559 00000100:00080000:2.0:1790772563.113512:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772563.113521:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772563.113531:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772563.113539:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534479360:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772563.113560:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534480000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772563.113604:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772563.113616:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772563.113627:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1790772563.113635:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772563/real 1790772563] req@ffff9c4efbcc1c00 x1877759534479360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772563.113636:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772563.113690:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1559 00000100:00080000:2.0:1790772563.113699:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772563.113711:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772563.113724:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772563.113729:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534479360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772563.113749:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534480128/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772563.113775:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772563.113782:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772563.113783:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772563.113789:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772563.113797:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772563.113803:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1559 00000100:00080000:2.0:1790772563.113811:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772563.113812:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534479360:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772563.113823:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772563.113831:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534479488:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790772563.113833:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772563.113859:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534480256/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772563.113877:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772563.113900:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772563/real 1790772563] req@ffff9c4efbcc2300 x1877759534479488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772563.113917:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534479488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772563.113929:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772563.113934:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772563.113939:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534479488:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772563.113947:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534479616:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772563.113968:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772563.113980:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772563/real 1790772563] req@ffff9c4efbcc1880 x1877759534479616/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772563.113995:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534479616/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772563.114008:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772563.114012:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772563.114018:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534479616:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772563.114026:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534479744:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772563.114046:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772563.114058:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772563/real 1790772563] req@ffff9c4efbcc1f80 x1877759534479744/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772563.114072:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534479744/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772563.114085:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772563.114089:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772563.114093:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534479744:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772563.114101:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534479872:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772563.114121:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772563.114133:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772563/real 1790772563] req@ffff9c4efbcc0700 x1877759534479872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772563.114148:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534479872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772563.114167:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772563.114171:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772563.114175:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534479872:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772563.114204:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534480000:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772563.114226:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772563.114239:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772563/real 1790772563] req@ffff9c4efbcc0a80 x1877759534480000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772563.114255:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534480000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772563.114266:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772563.114270:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772563.114274:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534480000:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772563.114281:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534480128:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772563.114301:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772563.114312:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772563/real 1790772563] req@ffff9c4efbcc3100 x1877759534480128/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772563.114327:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534480128/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772563.114341:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772563.114345:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772563.114349:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534480128:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772563.114356:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534480256:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772563.114375:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772563.114386:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772563/real 1790772563] req@ffff9c4efbcc0380 x1877759534480256/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772563.114400:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534480256/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772563.114415:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772563.114420:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772563.114424:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534480256:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772563.176752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772563.176754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772563.176815:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772563.176822:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772563.240750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772563.240819:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772563.240939:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772563.368786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772563.368798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772563.432792:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772563.496848:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772563.560776:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772563.560814:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772563.560838:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772563.624775:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772563.688813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772563.688814:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772563.688863:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772563.816743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772563.816767:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772563.816780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772563.816782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772563.880768:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772563.880803:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772563.880818:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772563.880828:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772563.880841:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772563.880881:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772564.008776:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772564.072748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772564.136742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772564.200817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772564.200817:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772564.200818:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772564.200818:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772564.264824:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772564.264869:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772564.264890:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772564.392776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772564.392801:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772564.456738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772564.520709:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772564.584708:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772564.584708:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772564.584748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772564.648710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772564.712722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772564.712736:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772564.712745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772564.840728:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772564.840736:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772564.840736:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772564.840739:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772564.904736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772564.904736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772564.904751:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772564.904755:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772564.904786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772564.904791:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772565.032757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772565.096828:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772565.160717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772565.224730:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772565.224741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772565.224743:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772565.224748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772565.288769:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772565.288807:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772565.288867:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772565.416730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772565.416730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772565.480711:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772565.544729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772565.608777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772565.608784:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772565.608806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772565.672769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772565.736799:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772565.736802:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772565.736809:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772565.864796:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772565.864796:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772565.864806:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772565.864807:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772565.928753:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772565.928765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772565.928794:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772565.928831:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772565.928844:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772565.928853:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772566.056775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772566.120794:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772566.184769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772566.248790:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772566.248797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772566.248815:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772566.248816:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772566.312784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772566.312843:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772566.312907:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772566.440764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772566.440800:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772566.504752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772566.568795:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772566.632775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772566.632788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772566.632794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772566.696746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772566.760777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772566.760787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772566.760787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772566.888783:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772566.888783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772566.888794:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772566.888799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772566.952790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772566.952815:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772566.952817:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772566.952833:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772566.952885:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772566.952895:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772567.080804:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772567.144759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772567.208740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772567.272753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772567.272767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772567.272790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772567.272790:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772567.336803:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772567.336847:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772567.336884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772567.464778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772567.464778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772567.528775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772567.592776:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772567.656754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772567.656776:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772567.656858:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772567.720766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772567.784775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772567.784787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772567.784818:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772567.912788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772567.912806:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772567.912806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772567.912828:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772567.976818:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772567.976819:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772567.976819:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772567.976835:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772567.976880:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772567.976887:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772568.104779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772568.168716:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772568.232761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772568.232806:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772568.232813:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772568.232818:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772568.232842:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1564 00000100:00080000:2.0:1790772568.232861:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772568.232876:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772568.232930:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534480384/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772568.232962:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772568.232967:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772568.232969:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772568.232973:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1564 00000100:00080000:2.0:1790772568.232977:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772568.232982:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772568.232994:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772568.233006:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534480512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772568.233017:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772568.233021:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772568.233023:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772568.233029:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1564 00000100:00080000:2.0:1790772568.233034:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772568.233039:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772568.233045:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772568.233062:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534480640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772568.233063:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534480384:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772568.233084:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772568.233089:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772568.233091:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772568.233095:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1564 00000100:00080000:2.0:1790772568.233100:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772568.233104:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772568.233111:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772568.233125:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534480768/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772568.233134:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772568.233138:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772568.233140:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772568.233145:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1564 00000100:00080000:2.0:1790772568.233149:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772568.233154:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772568.233158:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:3.0:1790772568.233166:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772568.233172:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534480896/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772568.233184:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772568.233188:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772568.233191:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772568.233196:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1564 00000100:00080000:2.0:1790772568.233199:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772568.233204:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772568.233210:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772568.233226:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534481024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772568.233233:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772568/real 1790772568] req@ffff9c4efbcc2a00 x1877759534480384/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772568.233246:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772568.233256:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772568.233259:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772568.233266:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1564 00000100:00100000:3.0:1790772568.233269:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534480384/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772568.233271:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772568.233283:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772568.233294:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1790772568.233298:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772568.233306:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772568.233310:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534481152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772568.233317:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534480384:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772568.233335:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772568.233336:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534480512:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772568.233344:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772568.233349:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772568.233355:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1564 00000100:00080000:2.0:1790772568.233361:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772568.233369:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:3.0:1790772568.233377:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790772568.233381:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772568.233401:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772568/real 1790772568] req@ffff9c4efbcc1500 x1877759534480512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772568.233402:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534481280/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772568.233453:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772568.233462:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1790772568.233462:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534480512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772568.233470:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772568.233500:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1564 00000100:00080000:2.0:1790772568.233510:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772568.233513:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772568.233521:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772568.233522:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772568.233532:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534480512:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790772568.233538:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772568.233563:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534481408/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772568.233573:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534480640:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772568.233602:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772568.233616:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772568/real 1790772568] req@ffff9c4efbcc0380 x1877759534480640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772568.233629:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534480640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772568.233638:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772568.233641:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772568.233645:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534480640:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772568.233681:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534480768:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772568.233701:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772568.233712:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772568/real 1790772568] req@ffff9c4efbcc3100 x1877759534480768/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772568.233724:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534480768/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772568.233735:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772568.233739:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772568.233743:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534480768:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772568.233748:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534480896:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772568.233763:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772568.233772:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772568/real 1790772568] req@ffff9c4efbcc0a80 x1877759534480896/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772568.233782:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534480896/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772568.233792:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772568.233796:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772568.233799:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534480896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772568.233805:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534481024:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772568.233819:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772568.233828:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772568/real 1790772568] req@ffff9c4efbcc0700 x1877759534481024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772568.233839:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534481024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772568.233855:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772568.233858:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772568.233862:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534481024:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772568.233869:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534481152:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772568.233884:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772568.233892:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772568/real 1790772568] req@ffff9c4efbcc1f80 x1877759534481152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772568.233903:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534481152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772568.233911:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772568.233914:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772568.233917:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534481152:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772568.233923:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534481280:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772568.233937:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772568.233946:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772568/real 1790772568] req@ffff9c4efbcc1880 x1877759534481280/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772568.233957:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534481280/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772568.233968:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772568.233971:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772568.233974:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534481280:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772568.233993:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534481408:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772568.234009:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772568.234018:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772568/real 1790772568] req@ffff9c4efbcc2300 x1877759534481408/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772568.234030:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534481408/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772568.234040:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772568.234043:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772568.234046:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534481408:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772568.296718:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772568.296718:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772568.296726:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772568.296727:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772568.360790:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772568.360825:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772568.360923:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772568.488775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772568.488793:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772568.552766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772568.616917:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772568.680796:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772568.680806:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772568.680888:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772568.744744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772568.808744:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772568.808786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772568.808800:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772568.936777:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772568.936790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772568.936804:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772568.936804:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772569.000804:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772569.000814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772569.000814:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772569.000828:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772569.000867:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772569.000872:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772569.129733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772569.192753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772569.256738:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772569.320786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772569.320786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772569.320806:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772569.320808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772569.384794:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772569.384806:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772569.384863:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772569.512814:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772569.512816:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772569.576778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772569.640778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772569.704750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772569.704815:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772569.704862:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772569.768841:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772569.832772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772569.832789:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772569.832790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772569.960768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772569.960789:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772569.960797:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772569.960807:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772570.024780:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772570.024792:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772570.024840:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772570.024841:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772570.024871:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772570.024893:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772570.153716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772570.216707:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772570.280740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772570.344774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772570.344805:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772570.344810:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772570.344914:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772570.408735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772570.408774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772570.408839:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772570.536762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772570.536774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772570.600796:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772570.664811:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772570.728804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772570.728830:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772570.728889:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772570.792762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772570.856772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772570.856802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772570.856806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772570.984772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772570.984779:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772570.984790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772570.984791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772571.048784:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772571.048787:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772571.048803:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772571.048811:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772571.048835:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772571.048889:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772571.177751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772571.240765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772571.304768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772571.368763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772571.368816:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772571.368816:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772571.368817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772571.432809:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772571.432871:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790772571.432896:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772571.560767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772571.560773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772571.624750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772571.688792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772571.752783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772571.752807:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772571.752809:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772571.816751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772571.880766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772571.880776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772571.880791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772572.008774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772572.008778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772572.008795:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772572.008799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772572.072771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772572.072792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772572.072810:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772572.072816:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772572.072839:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772572.072868:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772572.200783:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772572.264758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772572.328778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772572.392754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772572.392764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772572.392783:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772572.392787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772572.456769:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772572.456798:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772572.456992:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772572.584742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772572.584788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772572.648841:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772572.712768:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772572.776787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772572.776788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772572.776788:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772572.840751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772572.904789:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772572.904802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772572.904804:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772573.032771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772573.032796:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772573.032862:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772573.032878:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772573.096791:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772573.096795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772573.096845:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772573.096853:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772573.096895:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772573.096899:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772573.224755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772573.288720:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772573.352767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772573.352796:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772573.352808:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772573.352813:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772573.352827:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1569 00000100:00080000:0.0:1790772573.352842:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772573.352858:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772573.352908:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534481536/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772573.352948:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772573.352955:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772573.352958:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772573.352963:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1569 00000100:00080000:0.0:1790772573.352968:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772573.352974:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772573.352987:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772573.353007:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534481664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772573.353022:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772573.353030:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1790772573.353035:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534481536:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772573.353036:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772573.353047:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1569 00000100:00080000:0.0:1790772573.353057:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772573.353062:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772573.353069:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772573.353082:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534481792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772573.353095:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772573.353100:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772573.353103:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772573.353109:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1569 00000100:00080000:0.0:1790772573.353116:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772573.353123:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772573.353130:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772573.353145:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534481920/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772573.353161:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772573.353162:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772573.353173:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772573.353177:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772573.353184:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1569 00000100:00080000:0.0:1790772573.353202:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772573.353210:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772573.353217:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772573.353228:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772573/real 1790772573] req@ffff9c4ec7a7df80 x1877759534481536/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772573.353238:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534482048/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772573.353275:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772573.353281:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772573.353285:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772573.353287:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534481536/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772573.353296:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1569 00000100:00080000:0.0:1790772573.353306:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772573.353319:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772573.353322:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790772573.353326:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1790772573.353335:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772573.353348:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534481536:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772573.353361:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534482176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772573.353412:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772573.353416:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534481664:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772573.353419:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772573.353425:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772573.353437:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1569 00000100:00080000:0.0:1790772573.353442:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772573.353448:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772573.353455:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772573.353474:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534482304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772573.353488:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1790772573.353490:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772573.353498:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772573.353504:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772573.353511:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1569 00000100:00080000:0.0:1790772573.353517:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772573.353530:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772573/real 1790772573] req@ffff9c4ec7a7f480 x1877759534481664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772573.353531:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772573.353556:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772573.353584:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534481664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772573.353593:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534482432/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772573.353644:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772573.353695:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772573.353699:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772573.353708:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1569 00000100:00080000:3.0:1790772573.353711:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772573.353716:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772573.353724:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772573.353733:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772573.353740:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534481664:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790772573.353748:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772573.353757:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534481792:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772573.353774:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534482560/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772573.353805:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772573.353830:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772573/real 1790772573] req@ffff9c4ec7a7c380 x1877759534481792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772573.353848:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534481792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772573.353860:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772573.353865:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772573.353870:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534481792:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772573.353878:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534481920:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772573.353901:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772573.353913:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772573/real 1790772573] req@ffff9c4ec7a7d880 x1877759534481920/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772573.353929:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534481920/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772573.353941:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772573.353946:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772573.353950:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534481920:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772573.353957:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534482048:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772573.353986:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772573.354008:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772573/real 1790772573] req@ffff9c4ec7a7ed80 x1877759534482048/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772573.354033:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534482048/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772573.354053:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772573.354061:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772573.354069:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534482048:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772573.354098:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534482176:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772573.354128:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772573.354144:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772573/real 1790772573] req@ffff9c4ec7a7ca80 x1877759534482176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772573.354169:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534482176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772573.354193:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772573.354200:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772573.354207:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534482176:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772573.354219:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534482304:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772573.354259:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772573.354283:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772573/real 1790772573] req@ffff9c4ec7a7e300 x1877759534482304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772573.354304:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534482304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772573.354316:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772573.354320:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772573.354326:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534482304:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772573.354339:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534482432:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772573.354403:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772573.354430:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772573/real 1790772573] req@ffff9c4ec7a7fb80 x1877759534482432/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772573.354458:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534482432/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772573.354482:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772573.354487:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772573.354492:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534482432:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772573.354501:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534482560:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772573.354530:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772573.354545:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772573/real 1790772573] req@ffff9c4ec7a7c700 x1877759534482560/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772573.354560:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534482560/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772573.354572:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772573.354577:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772573.354583:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534482560:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772573.416783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772573.416798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772573.416807:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772573.416813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772573.480759:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772573.480843:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772573.481011:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772573.608790:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772573.608797:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772573.672767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772573.736773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772573.800779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772573.800837:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772573.800839:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772573.864749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772573.928839:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772573.928859:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772573.928867:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772574.056780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772574.056850:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772574.056853:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772574.056855:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772574.120771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772574.120794:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772574.120796:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772574.120845:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772574.120887:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772574.120888:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772574.248775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772574.312742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772574.376767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772574.440737:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772574.440789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772574.440794:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772574.440794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772574.504781:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772574.504781:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772574.504789:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772574.632727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772574.632736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772574.696707:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772574.760699:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772574.824736:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772574.824736:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772574.824737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772574.888707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772574.952736:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772574.952770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772574.952770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772575.080724:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772575.080731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772575.080751:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772575.080752:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772575.144728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772575.144757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772575.144778:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772575.144791:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772575.144809:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772575.144810:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772575.272782:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772575.336749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772575.400713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772575.464720:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772575.464803:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772575.464805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772575.464807:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772575.528726:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772575.528741:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772575.528839:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772575.656753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772575.656771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772575.720738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772575.784915:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772575.848787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772575.848807:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772575.848808:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772575.912787:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772575.976790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772575.976834:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772575.976843:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772576.104791:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772576.104859:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772576.104860:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772576.104863:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772576.168785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772576.168800:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772576.168825:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772576.168832:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772576.168875:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772576.168910:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772576.296766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772576.360751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772576.424784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772576.488776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772576.488788:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772576.488816:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772576.488816:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772576.552771:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772576.552847:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772576.552876:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772576.680787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772576.680794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772576.744745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772576.808778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772576.872760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772576.872770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772576.872792:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772576.936759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772577.000758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772577.000794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772577.000818:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772577.128783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772577.128836:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772577.128838:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772577.128861:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772577.192792:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772577.192795:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772577.192845:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772577.192848:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772577.192861:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772577.192909:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772577.320760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772577.384779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772577.448795:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772577.512771:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772577.512801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772577.512816:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772577.512827:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772577.576848:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772577.576875:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772577.576940:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772577.704763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772577.704812:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772577.768745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772577.832782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772577.896826:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772577.896836:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772577.896879:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772577.960745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772578.024763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772578.024813:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772578.024847:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772578.152771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772578.152786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772578.152820:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772578.152834:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772578.216780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772578.216790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772578.216815:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772578.216851:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772578.216881:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772578.216882:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772578.344765:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772578.408772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772578.472788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772578.472849:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772578.472860:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772578.472867:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772578.472882:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1574 00000100:00080000:1.0:1790772578.472900:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772578.472919:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772578.472985:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534482688/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772578.473033:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772578.473042:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772578.473048:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772578.473058:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1574 00000100:00080000:1.0:1790772578.473065:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772578.473076:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772578.473097:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772578.473123:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534482688:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772578.473128:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534482816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772578.473149:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772578.473156:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772578.473160:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772578.473170:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1574 00000100:00080000:1.0:1790772578.473176:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772578.473186:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772578.473195:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772578.473215:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534482944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772578.473232:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772578.473239:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772578.473244:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1790772578.473244:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772578.473253:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1574 00000100:00080000:1.0:1790772578.473262:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772578.473307:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772578.473319:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772578.473341:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534483072/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772578.473360:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772578.473362:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772578/real 1790772578] req@ffff9c4efac4fb80 x1877759534482688/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772578.473367:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772578.473371:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772578.473380:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1574 00000100:00080000:1.0:1790772578.473387:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772578.473397:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772578.473399:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534482688/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772578.473407:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:3.0:1790772578.473421:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790772578.473425:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534483200/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772578.473431:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772578.473441:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534482688:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772578.473444:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772578.473451:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772578.473454:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772578.473462:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1574 00000100:00080000:1.0:1790772578.473468:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772578.473477:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772578.473482:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534482816:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790772578.473486:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772578.473504:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534483328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772578.473525:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1790772578.473527:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772578.473532:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772578.473537:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772578.473546:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1574 00000100:00100000:3.0:1790772578.473552:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772578/real 1790772578] req@ffff9c4efac4df80 x1877759534482816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772578.473553:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772578.473563:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772578.473573:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772578.473578:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534482816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772578.473593:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534483456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772578.473610:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772578.473613:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772578.473617:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772578.473620:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772578.473622:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772578.473628:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534482816:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772578.473631:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1574 00000100:00080000:1.0:1790772578.473637:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772578.473641:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534482944:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772578.473646:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772578.473710:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790772578.473736:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534483584/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772578.473744:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772578.473756:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772578.473763:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772578.473768:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772578.473769:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772578/real 1790772578] req@ffff9c4efac4e300 x1877759534482944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772578.473777:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1574 00000100:00080000:1.0:1790772578.473785:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772578.473795:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772578.473805:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772578.473810:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534482944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772578.473825:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534483712/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772578.473830:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772578.473838:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772578.473845:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534482944:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772578.473858:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534483072:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772578.473900:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772578.473925:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772578/real 1790772578] req@ffff9c4efac4f480 x1877759534483072/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772578.473947:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534483072/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772578.473961:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772578.473966:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772578.473971:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534483072:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772578.473980:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534483200:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772578.474009:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772578.474022:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772578/real 1790772578] req@ffff9c4efac4f800 x1877759534483200/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772578.474037:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534483200/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772578.474059:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772578.474064:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772578.474068:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534483200:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772578.474092:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534483328:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772578.474114:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772578.474127:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772578/real 1790772578] req@ffff9c4efac4dc00 x1877759534483328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772578.474141:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534483328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772578.474163:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772578.474167:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772578.474172:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534483328:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772578.474180:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534483456:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772578.474199:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772578.474211:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772578/real 1790772578] req@ffff9c4efac4ed80 x1877759534483456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772578.474225:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534483456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772578.474236:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772578.474240:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772578.474246:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534483456:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772578.474253:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534483584:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772578.474304:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772578.474328:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772578/real 1790772578] req@ffff9c4efac4c700 x1877759534483584/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772578.474355:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534483584/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772578.474377:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772578.474382:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772578.474387:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534483584:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772578.474395:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534483712:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772578.474426:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772578.474440:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772578/real 1790772578] req@ffff9c4efac4ea00 x1877759534483712/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772578.474456:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534483712/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772578.474469:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772578.474473:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772578.474478:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534483712:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772578.536758:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772578.536809:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772578.536815:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772578.536840:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772578.600758:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772578.600807:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772578.600975:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772578.728772:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772578.728775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772578.792747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772578.856821:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772578.920771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772578.920789:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772578.920789:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772578.984753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772579.048778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772579.048784:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772579.048784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772579.176772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772579.176778:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772579.176781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772579.176809:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772579.240772:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772579.240807:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772579.240810:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772579.240812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772579.240846:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772579.240866:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772579.368764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772579.432749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772579.496778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772579.560832:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772579.560836:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772579.560839:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772579.560854:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772579.624868:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772579.624961:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772579.625036:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772579.752821:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772579.752839:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772579.816748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772579.880770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772579.944765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772579.944771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772579.944801:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772580.008763:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772580.072758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772580.072758:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772580.072795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772580.200727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772580.200727:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772580.200727:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772580.200742:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772580.264727:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772580.264730:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772580.264731:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772580.264742:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772580.264758:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772580.264762:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772580.392752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772580.456704:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772580.520716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772580.584719:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772580.584719:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772580.584723:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772580.584737:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772580.648747:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772580.648747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772580.648790:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772580.776713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772580.776713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772580.840713:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772580.904708:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772580.968770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772580.968781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772580.968785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772581.032760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772581.096765:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772581.096765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772581.096765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772581.224803:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772581.224803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772581.224803:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772581.224832:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772581.288768:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772581.288792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772581.288836:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772581.288840:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772581.288863:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772581.288881:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772581.416775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772581.480752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772581.544738:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772581.608750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772581.608783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772581.608793:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772581.608806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772581.672798:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772581.672830:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772581.672849:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772581.800788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772581.800789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772581.864763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772581.928801:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772581.992768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772581.992799:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772581.992800:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772582.056776:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772582.120755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772582.120777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772582.120809:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772582.248762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772582.248773:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772582.248784:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772582.248811:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772582.312781:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772582.312808:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772582.312819:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772582.312873:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772582.312899:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772582.312911:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772582.440746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772582.504762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772582.568770:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772582.632757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772582.632766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772582.632806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772582.632807:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772582.696804:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772582.696888:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772582.696910:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772582.824757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772582.824775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772582.888766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772582.952879:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772583.016788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772583.016788:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772583.016788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772583.080729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772583.144760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772583.144779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772583.144782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772583.272781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772583.272789:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772583.272811:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772583.272829:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772583.336797:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772583.336798:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772583.336809:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772583.336831:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772583.336858:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772583.336871:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772583.464751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772583.528741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772583.592747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772583.592780:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772583.592791:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772583.592800:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772583.592819:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1579 00000100:00080000:1.0:1790772583.592842:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772583.592866:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772583.592945:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534483840/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772583.592991:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772583.592997:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772583.593000:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772583.593006:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1579 00000100:00080000:1.0:1790772583.593013:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772583.593022:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772583.593038:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772583.593061:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534483968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772583.593079:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534483840:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772583.593080:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772583.593097:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772583.593100:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772583.593108:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1579 00000100:00080000:1.0:1790772583.593113:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772583.593119:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772583.593127:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772583.593142:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534484096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772583.593166:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772583.593172:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772583.593207:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772583.593225:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1579 00000100:00080000:1.0:1790772583.593233:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1790772583.593240:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772583.593250:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772583.593263:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772583.593293:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534484224/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772583.593316:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772583.593325:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1790772583.593332:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772583/real 1790772583] req@ffff9c4efac4e680 x1877759534483840/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772583.593334:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772583.593355:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1579 00000100:00080000:1.0:1790772583.593368:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772583.593378:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534483840/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772583.593381:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772583.593391:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772583.593412:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534484352/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772583.593436:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772583.593439:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772583.593451:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772583.593458:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772583.593459:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772583.593473:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1579 00000100:00100000:3.0:1790772583.593482:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534483840:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772583.593483:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772583.593497:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772583.593504:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772583.593519:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534484480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772583.593533:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534483968:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772583.593536:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772583.593550:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772583.593552:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772583.593560:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1579 00000100:00080000:1.0:1790772583.593565:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772583.593573:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772583.593581:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1790772583.593591:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772583.593601:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534484608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772583.593616:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772583.593619:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772583/real 1790772583] req@ffff9c4efac4ea00 x1877759534483968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772583.593627:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772583.593633:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772583.593647:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1579 00000100:00100000:3.0:1790772583.593700:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534483968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772583.593701:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772583.593716:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772583.593731:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790772583.593754:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534484736/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772583.593762:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772583.593779:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772583.593794:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534483968:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772583.593797:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772583.593811:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1790772583.593816:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534484096:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772583.593819:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772583.593838:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1579 00000100:00080000:1.0:1790772583.593848:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772583.593866:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:3.0:1790772583.593872:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772583.593889:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772583.593900:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772583/real 1790772583] req@ffff9c4efac4c700 x1877759534484096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772583.593933:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534484096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772583.593934:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534484864/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772583.593983:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772583.593992:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772583.594001:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534484096:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772583.594018:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534484224:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772583.594065:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772583.594090:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772583/real 1790772583] req@ffff9c4efac4ed80 x1877759534484224/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772583.594117:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534484224/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772583.594140:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772583.594148:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772583.594156:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534484224:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772583.594170:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534484352:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772583.594236:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772583.594255:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772583/real 1790772583] req@ffff9c4efac4dc00 x1877759534484352/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772583.594272:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534484352/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772583.594286:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772583.594290:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772583.594295:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534484352:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772583.594322:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534484480:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772583.594347:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772583.594369:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772583/real 1790772583] req@ffff9c4efac4f800 x1877759534484480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772583.594394:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534484480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772583.594425:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772583.594434:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772583.594442:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534484480:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772583.594455:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534484608:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772583.594498:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772583.594523:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772583/real 1790772583] req@ffff9c4efac4f480 x1877759534484608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772583.594549:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534484608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772583.594562:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772583.594567:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772583.594572:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534484608:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772583.594581:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534484736:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772583.594610:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772583.594625:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772583/real 1790772583] req@ffff9c4efac4e300 x1877759534484736/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772583.594640:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534484736/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772583.594684:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772583.594689:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772583.594694:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534484736:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772583.594702:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534484864:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772583.594727:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772583.594741:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772583/real 1790772583] req@ffff9c4efac4df80 x1877759534484864/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772583.594757:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534484864/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772583.594770:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772583.594774:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772583.594780:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534484864:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772583.656761:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772583.656763:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772583.656791:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772583.656806:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772583.720775:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772583.720813:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772583.720950:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772583.848745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772583.848762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772583.912797:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772583.976783:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772584.040782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772584.040782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772584.040791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772584.104802:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772584.168743:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772584.168778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772584.168799:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772584.296758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772584.296778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772584.296791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772584.296811:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772584.360774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772584.360812:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772584.360819:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772584.360842:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772584.360871:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772584.360889:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772584.488756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772584.552766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772584.616768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772584.680757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772584.680791:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772584.680794:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772584.680815:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772584.744901:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772584.744931:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772584.745057:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772584.872830:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772584.872841:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772584.936761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772585.000803:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772585.064819:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772585.064821:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772585.064862:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772585.128754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772585.192772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772585.192798:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772585.192816:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772585.320837:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772585.320882:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772585.320885:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772585.320909:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772585.384762:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772585.384777:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772585.384799:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772585.384849:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772585.384860:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772585.384889:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772585.512760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772585.576767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772585.640750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772585.704782:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772585.704808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772585.704839:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772585.704842:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772585.768768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772585.768816:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772585.768889:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772585.896759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772585.896796:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772585.960774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772586.024749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772586.088770:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772586.088819:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772586.088821:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772586.152832:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772586.216766:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772586.216788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772586.216812:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772586.344778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772586.344828:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772586.344838:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772586.344842:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772586.408802:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772586.408845:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772586.408859:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772586.408869:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772586.408909:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772586.408919:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772586.536751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772586.600751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772586.664757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772586.728772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772586.728819:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772586.728863:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772586.728872:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772586.792847:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772586.792876:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772586.792954:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772586.920778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772586.920792:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772586.984765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772587.048800:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772587.112726:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772587.112755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772587.112801:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772587.176760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772587.240825:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772587.240854:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772587.240855:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772587.368775:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772587.368794:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772587.368830:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772587.368831:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772587.432800:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772587.432846:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772587.432847:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772587.432859:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772587.432911:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772587.432951:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772587.560775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772587.624805:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772587.688781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772587.752762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772587.752813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772587.752813:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772587.752852:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772587.816767:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772587.816813:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772587.817021:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772587.944766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772587.944783:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772588.008772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772588.072801:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772588.136774:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772588.136788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772588.136811:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772588.200748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772588.264774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772588.264781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772588.264817:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772588.392754:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772588.392797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772588.392805:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772588.392827:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772588.456766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772588.456801:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772588.456806:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772588.456838:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772588.456849:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772588.456872:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772588.584756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772588.648769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790772588.712811:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772588.712824:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772588.712832:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772588.712841:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772588.712851:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1584 00000100:00080000:2.0:1790772588.712871:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772588.712892:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772588.712974:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534484992/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772588.713014:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772588.713019:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772588.713022:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772588.713029:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1584 00000100:00080000:2.0:1790772588.713036:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772588.713043:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772588.713061:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772588.713069:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534484992:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772588.713118:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534485120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772588.713136:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772588.713141:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772588.713144:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772588.713151:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1584 00000100:00080000:2.0:1790772588.713157:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772588.713164:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772588.713172:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772588.713190:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534485248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772588.713203:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772588.713209:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772588.713212:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772588.713217:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1584 00000100:00080000:2.0:1790772588.713223:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772588.713230:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772588.713238:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772588.713254:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534485376/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772588.713268:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772588.713274:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772588.713278:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772588.713284:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1584 00000100:00080000:2.0:1790772588.713290:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772588.713292:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534485120:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772588.713302:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772588.713312:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772588.713332:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534485504/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772588.713349:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772588.713356:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534485248:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772588.713358:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772588.713366:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772588.713376:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1584 00000100:00080000:2.0:1790772588.713383:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772588.713403:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772588.713410:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534485376:192.168.204.150@tcp:8:kworker.0 00000008:00100000:2.0:1790772588.713416:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772588.713438:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534485632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772588.713454:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772588.713460:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1790772588.713463:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534485504:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772588.713466:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772588.713483:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1584 00000100:00080000:2.0:1790772588.713491:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772588.713500:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772588.713508:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772588.713516:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534485632:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772588.713534:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534485760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772588.713555:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772588.713561:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534485760:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772588.713580:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772588.713584:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772588.713593:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1584 00000100:00080000:2.0:1790772588.713601:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772588.713610:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772588.713617:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772588.713635:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534485888/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772588.713684:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772588.713690:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772588.713694:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772588.713700:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534485888:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772588.713705:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1584 00000100:00080000:2.0:1790772588.713710:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772588.713719:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772588.713732:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772588.713761:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534486016/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772588.713785:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534486016:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772588.776807:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772588.776817:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772588.776817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772588.776830:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772588.840802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772588.840882:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772588.840930:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772588.968757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772588.968786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772589.032745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772589.096750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772589.160773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772589.160801:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772589.160804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772589.224790:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772589.288761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772589.288779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772589.288795:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772589.416764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772589.416785:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772589.416801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772589.416805:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772589.480789:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772589.480807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772589.480809:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772589.480833:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772589.480859:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772589.480887:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772589.608750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772589.672740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772589.736751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772589.800751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772589.800780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772589.800795:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772589.800795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772589.864810:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772589.864867:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772589.864877:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772589.992832:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772589.992847:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772590.056737:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772590.120904:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772590.184774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772590.184799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772590.184802:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772590.248749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772590.312805:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772590.312806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772590.312812:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772590.440783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772590.440804:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772590.440806:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772590.440807:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772590.504778:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772590.504811:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772590.504818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772590.504818:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772590.504865:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772590.504865:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772590.632760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772590.696767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772590.760773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772590.824780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772590.824788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772590.824789:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772590.824794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772590.888761:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772590.888811:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772590.888921:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772591.016806:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772591.016843:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772591.080774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772591.144782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772591.208771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772591.208791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772591.208795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772591.272866:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772591.336776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772591.336789:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772591.336810:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772591.464771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772591.464784:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772591.464829:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772591.464837:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772591.528796:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772591.528813:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772591.528876:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772591.528876:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772591.528879:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772591.528899:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772591.656751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772591.720784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772591.784812:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:0.0:1790772591.785112:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:0.0:1790772591.785176:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1790772591.785228:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772591.785322:0:2385:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:0.0:1790772591.785354:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772591.785377:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772591.785397:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772591.785421:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:3.0:1790772591.785423:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772588/real 1790772591] req@ffff9c4efbcc0380 x1877759534486016/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1790772591.785447:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772591.785476:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:3.0:1790772591.785500:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534486016/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1790772591.785509:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:3.0:1790772591.785547:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772591.785561:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772591.785572:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534486016:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772591.785591:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772588/real 1790772591] req@ffff9c4efbcc3100 x1877759534485888/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772591.785636:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534485888/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772591.785695:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772591.785703:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772591.785711:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534485888:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772591.785756:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772588/real 1790772591] req@ffff9c4efbcc0a80 x1877759534485760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772591.785784:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534485760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772591.785801:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772591.785810:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772591.785819:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534485760:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772591.785833:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772588/real 1790772591] req@ffff9c4efbcc1f80 x1877759534485504/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772591.785858:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534485504/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772591.785889:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772591.785897:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772591.785904:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534485504:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772591.785918:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772588/real 1790772591] req@ffff9c4efbcc0700 x1877759534485632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772591.785943:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534485632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772591.785973:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772591.785981:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772591.785988:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534485632:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772591.786003:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772588/real 1790772591] req@ffff9c4efbcc2680 x1877759534485120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772591.786060:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534485120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772591.786089:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772591.786096:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772591.786103:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534485120:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772591.786116:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772588/real 1790772591] req@ffff9c4efbcc2300 x1877759534485248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772591.786138:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534485248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772591.787518:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772591.787549:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772591.787559:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534485248:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772591.787592:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772588/real 1790772591] req@ffff9c4efbcc1880 x1877759534485376/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772591.787635:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534485376/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772591.787685:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772591.787696:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772591.787705:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534485376:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772591.787722:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772588/real 1790772591] req@ffff9c4efbcc1c00 x1877759534484992/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772591.787751:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534484992/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772591.787769:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772591.787778:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772591.787785:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534484992:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772591.848799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772591.848804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772591.848820:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772591.848821:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772591.912834:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772591.912896:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772591.912910:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772592.040783:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772592.040786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772592.104780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772592.168769:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772592.232758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772592.232760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772592.232793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772592.296770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772592.360770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772592.360775:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772592.360777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772592.488754:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772592.488785:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772592.488791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772592.488803:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772592.552761:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772592.552804:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772592.552804:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772592.552813:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772592.552841:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772592.552863:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772592.680769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772592.744764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772592.808755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772592.872774:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772592.872806:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772592.872807:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772592.872809:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772592.936802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772592.936861:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772592.936910:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772593.064776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772593.064781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772593.128752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772593.192813:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772593.256759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772593.256786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772593.256792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772593.320761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772593.384805:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772593.384809:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772593.384809:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772593.512777:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772593.512779:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772593.512820:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772593.512863:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772593.576776:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772593.576837:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772593.576855:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772593.576874:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772593.576875:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772593.576927:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772593.704788:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772593.768737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772593.832812:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772593.832921:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772593.832933:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772593.832940:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772593.832956:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1589 00000100:00080000:0.0:1790772593.833000:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772593.833027:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772593.833102:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534486144/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772593.833163:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772593.833171:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772593.833175:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772593.833205:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1589 00000100:00080000:0.0:1790772593.833216:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772593.833228:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772593.833250:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772593.833268:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534486144:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772593.833287:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534486272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772593.833317:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772593.833325:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772593.833331:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772593.833341:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1589 00000100:00080000:0.0:1790772593.833353:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1790772593.833392:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772593.833396:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772593.833414:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772593.833447:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534486400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772593.833471:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772593.833479:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772593.833484:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772593.833497:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1589 00000100:00080000:0.0:1790772593.833507:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772593.833530:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772593/real 1790772593] req@ffff9c4ec7a7d180 x1877759534486144/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772593.833531:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772593.833551:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772593.833596:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534486528/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772593.833603:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534486144/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772593.833716:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772593.833724:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772593.833731:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772593.833762:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1589 00000100:00080000:3.0:1790772593.833772:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772593.833775:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772593.833797:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772593.833800:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772593.833818:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534486144:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790772593.833820:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772593.833878:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534486656/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772593.833892:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534486272:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772593.833927:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772593.833938:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772593.833945:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772593.833960:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1589 00000100:00080000:0.0:1790772593.833999:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1790772593.834014:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772593.834022:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772593.834039:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772593.834068:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534486784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772593.834087:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772593.834093:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772593.834097:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772593.834106:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1589 00000100:00080000:0.0:1790772593.834114:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772593.834123:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772593.834132:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772593.834151:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534486912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772593.834172:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772593.834178:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772593.834182:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772593.834190:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1589 00000100:00080000:0.0:1790772593.834196:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772593.834205:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772593.834215:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772593.834237:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534487040/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772593.834254:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772593.834260:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772593.834265:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772593.834274:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1589 00000100:00080000:0.0:1790772593.834282:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772593.834292:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772593.834303:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772593.834326:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534487168/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772593.834556:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772593/real 1790772593] req@ffff9c4ec7a7c700 x1877759534486272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772593.834592:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534486272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772593.834626:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772593.834634:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772593.834642:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534486272:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772593.834701:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534486400:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772593.834762:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772593.834790:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772593/real 1790772593] req@ffff9c4ec7a7fb80 x1877759534486400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772593.834817:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534486400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772593.834838:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772593.834847:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772593.834855:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534486400:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772593.834870:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534486528:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772593.834917:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772593.834943:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772593/real 1790772593] req@ffff9c4ec7a7e300 x1877759534486528/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772593.835137:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534486528/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772593.835160:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772593.835168:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772593.835176:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534486528:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772593.835218:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534486656:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772593.835269:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772593.835295:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772593/real 1790772593] req@ffff9c4ec7a7ca80 x1877759534486656/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772593.835324:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534486656/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772593.835346:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772593.835354:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772593.835361:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534486656:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772593.835375:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534486784:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772593.835418:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772593.835442:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772593/real 1790772593] req@ffff9c4ec7a7ed80 x1877759534486784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772593.835467:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534486784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772593.835496:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772593.835504:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772593.835512:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534486784:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772593.835526:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534486912:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772593.835569:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772593.835595:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772593/real 1790772593] req@ffff9c4ec7a7d880 x1877759534486912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772593.835621:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534486912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772593.835640:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772593.835647:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772593.835697:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534486912:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772593.835713:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534487040:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772593.835751:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772593.835768:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772593/real 1790772593] req@ffff9c4ec7a7c380 x1877759534487040/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772593.835785:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534487040/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772593.835799:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772593.835804:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772593.835809:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534487040:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772593.835817:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534487168:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772593.835837:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772593.835849:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772593/real 1790772593] req@ffff9c4ec7a7f480 x1877759534487168/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772593.835864:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534487168/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772593.835877:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772593.835881:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772593.835886:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534487168:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772593.896789:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772593.896793:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772593.896830:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772593.896841:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772593.960783:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772593.960831:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772593.960968:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772594.088766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772594.088798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772594.152753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772594.216750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772594.280748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772594.280777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772594.280813:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772594.344749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772594.408779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772594.408795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772594.408814:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772594.536824:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772594.536824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772594.536831:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772594.536832:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772594.600764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772594.600794:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772594.600866:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772594.600873:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772594.600876:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772594.600883:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772594.728764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772594.792752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772594.856813:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772594.920752:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772594.920799:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772594.920801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772594.920820:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772594.984766:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772594.984791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772594.984879:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772595.112780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772595.112798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772595.176751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772595.240752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772595.304769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772595.304793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772595.304807:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772595.368749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772595.432767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772595.432774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772595.432804:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772595.560767:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772595.560798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772595.560802:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772595.560824:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772595.624803:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772595.624805:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772595.624811:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772595.624835:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772595.624868:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772595.624879:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772595.752781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772595.816772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772595.880758:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772595.944782:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772595.944787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772595.944841:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772595.944848:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772596.008782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772596.008881:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772596.008930:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772596.136750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772596.136794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772596.200776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772596.264760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772596.328763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772596.328906:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772596.328909:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772596.392757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772596.456761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772596.456782:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772596.456799:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772596.584807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772596.584883:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772596.584886:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772596.584908:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772596.648838:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772596.648846:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772596.648899:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772596.648901:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772596.648963:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772596.649002:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772596.776740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772596.840768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772596.904792:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772596.968806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772596.968853:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772596.968870:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772596.968895:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772597.032809:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772597.032826:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772597.032969:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772597.160747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772597.160785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772597.224759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772597.288886:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772597.352766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772597.352781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772597.352805:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772597.416786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772597.480831:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772597.480854:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772597.480879:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772597.608795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772597.608799:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772597.608828:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772597.608844:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772597.672757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772597.672766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772597.672817:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772597.672860:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772597.672872:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772597.672888:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772597.800751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772597.864820:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772597.928763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772597.992795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772597.992813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772597.992852:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772597.992861:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772598.056867:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772598.056933:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772598.057020:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772598.184776:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772598.184789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772598.248760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772598.312750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772598.376762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772598.376771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772598.376800:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772598.440936:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772598.504794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772598.504824:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772598.504843:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772598.632763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772598.632817:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772598.632818:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772598.632831:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772598.696861:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772598.696911:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772598.696928:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772598.696981:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772598.697082:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772598.697113:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772598.824763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772598.888769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772598.952802:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772598.952828:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772598.952842:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772598.952851:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772598.952867:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1594 00000100:00080000:0.0:1790772598.952913:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772598.952935:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772598.953002:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534487296/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772598.953048:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772598.953054:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772598.953057:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772598.953063:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1594 00000100:00080000:0.0:1790772598.953068:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772598.953075:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772598.953092:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772598.953111:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534487296:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772598.953116:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534487424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772598.953138:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772598.953143:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772598.953146:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772598.953152:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1594 00000100:00080000:0.0:1790772598.953156:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772598.953163:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772598.953172:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772598.953190:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534487552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772598.953203:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772598.953208:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772598.953214:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772598.953221:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1594 00000100:00080000:0.0:1790772598.953226:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1790772598.953237:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772598.953238:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772598.953251:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772598.953270:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534487680/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772598.953283:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772598.953290:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772598.953293:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772598.953300:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1594 00000100:00080000:0.0:1790772598.953306:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772598.953318:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772598.953318:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772598/real 1790772598] req@ffff9c4ec7a7df80 x1877759534487296/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790772598.953331:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772598.953356:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534487808/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772598.953366:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534487296/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772598.953381:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772598.953394:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772598.953398:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772598.953399:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772598.953408:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1594 00000100:00080000:3.0:1790772598.953412:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772598.953416:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772598.953425:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534487296:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772598.953432:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772598.953442:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772598.953456:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534487936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772598.953471:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534487424:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772598.953474:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772598.953489:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772598.953493:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772598.953501:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1594 00000100:00080000:0.0:1790772598.953509:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772598.953517:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:3.0:1790772598.953529:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790772598.953532:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772598.953553:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534488064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772598.953568:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772598.953571:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772598/real 1790772598] req@ffff9c4ec7a7f480 x1877759534487424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772598.953574:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772598.953580:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772598.953593:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1594 00000100:00080000:0.0:1790772598.953604:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772598.953616:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772598.953625:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772598.953633:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534487424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772598.953645:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534488192/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772598.953704:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772598.953713:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772598.953718:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772598.953720:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772598.953736:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772598.953740:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1594 00000100:00100000:3.0:1790772598.953748:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534487424:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772598.953756:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772598.953763:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534487552:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772598.953777:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772598.953792:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:3.0:1790772598.953812:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772598.953827:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534488320/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772598.953837:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772598/real 1790772598] req@ffff9c4ec7a7c380 x1877759534487552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772598.953866:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534487552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772598.953911:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772598.953917:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772598.953923:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534487552:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772598.953932:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534487680:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772598.953964:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772598.953979:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772598/real 1790772598] req@ffff9c4ec7a7d880 x1877759534487680/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772598.954004:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534487680/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772598.954018:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772598.954023:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772598.954028:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534487680:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772598.954035:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534487808:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772598.954055:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772598.954067:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772598/real 1790772598] req@ffff9c4ec7a7ed80 x1877759534487808/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772598.954082:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534487808/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772598.954095:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772598.954098:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772598.954103:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534487808:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772598.954126:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534487936:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772598.954148:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772598.954161:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772598/real 1790772598] req@ffff9c4ec7a7ca80 x1877759534487936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772598.954176:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534487936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772598.954205:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772598.954210:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772598.954214:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534487936:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772598.954222:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534488064:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772598.954240:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772598.954252:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772598/real 1790772598] req@ffff9c4ec7a7e300 x1877759534488064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772598.954266:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534488064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772598.954277:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772598.954282:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772598.954286:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534488064:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772598.954293:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534488192:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772598.954311:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772598.954322:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772598/real 1790772598] req@ffff9c4ec7a7fb80 x1877759534488192/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772598.954337:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534488192/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772598.954350:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772598.954354:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772598.954360:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534488192:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772598.954368:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534488320:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772598.954387:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772598.954398:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772598/real 1790772598] req@ffff9c4ec7a7c700 x1877759534488320/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772598.954412:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534488320/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772598.954425:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772598.954429:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772598.954433:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534488320:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772599.016799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772599.016856:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772599.016912:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772599.016939:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772599.080795:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772599.080829:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772599.080972:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772599.208761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772599.208765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772599.272756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772599.336776:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772599.400762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772599.400768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772599.400804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772599.464770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772599.528820:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772599.528858:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772599.528861:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772599.656789:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772599.656798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772599.656897:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772599.656904:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772599.720832:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772599.720902:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772599.720924:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772599.720926:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772599.720930:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772599.720995:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772599.848776:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772599.912753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772599.976765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772600.040784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772600.040807:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772600.040812:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772600.040843:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772600.104782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772600.104824:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772600.105046:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772600.232772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772600.232816:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772600.296746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772600.360800:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772600.424769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772600.424776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772600.424801:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772600.488783:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772600.552838:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772600.552894:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772600.552903:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772600.680769:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772600.680821:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772600.680822:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772600.680867:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772600.744799:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772600.744867:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772600.744889:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772600.744911:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772600.744935:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772600.744972:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772600.872820:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772600.936741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772601.000742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772601.064810:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772601.064811:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772601.064818:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772601.064884:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772601.128809:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772601.128820:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772601.129067:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772601.256769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772601.256778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772601.320746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772601.384804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772601.448822:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772601.448853:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772601.448874:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772601.512737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772601.576776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772601.576813:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772601.576813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772601.704813:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772601.704813:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772601.704871:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772601.704873:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772601.768793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772601.768804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772601.768844:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772601.768864:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772601.768893:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772601.768933:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772601.896763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772601.960757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772602.024743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772602.088774:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772602.088886:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772602.088891:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772602.088927:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772602.152775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772602.152891:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772602.152995:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772602.280765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772602.280774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772602.344756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772602.408749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772602.472785:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772602.472801:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772602.472859:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772602.536752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772602.600830:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772602.600838:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772602.600868:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772602.728839:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772602.728857:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772602.728872:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772602.728884:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772602.792776:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772602.792791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772602.792830:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772602.792834:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772602.792874:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772602.792891:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772602.920742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772602.984772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772603.048751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772603.112757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772603.112839:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772603.112851:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772603.112870:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772603.176877:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772603.176939:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772603.177072:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772603.304767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772603.304775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772603.368758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772603.432828:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772603.496770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772603.496832:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772603.496833:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772603.560811:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772603.624763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772603.624782:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772603.624822:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772603.752752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772603.752814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772603.752820:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772603.752852:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772603.816765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772603.816771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772603.816860:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772603.816864:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772603.816891:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772603.816911:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772603.944751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772604.008746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772604.072771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772604.072802:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772604.072814:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772604.072821:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772604.072839:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1599 00000100:00080000:0.0:1790772604.072856:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772604.072877:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772604.072943:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534488448/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772604.072989:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772604.072997:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772604.073002:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772604.073013:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1599 00000100:00080000:0.0:1790772604.073022:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772604.073034:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772604.073048:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534488448:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790772604.073056:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772604.073096:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534488576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772604.073132:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772604.073138:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772604.073141:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772604.073149:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1599 00000100:00080000:0.0:1790772604.073157:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1790772604.073168:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772604.073169:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772604.073184:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772604.073208:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534488704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772604.073221:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772604.073227:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772604.073230:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772604.073235:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1599 00000100:00080000:0.0:1790772604.073240:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772604.073247:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772604.073255:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772604.073264:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772604/real 1790772604] req@ffff9c4ec7a7d180 x1877759534488448/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772604.073279:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534488832/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772604.073297:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772604.073306:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1790772604.073307:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534488448/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772604.073310:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772604.073326:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1599 00000100:00080000:0.0:1790772604.073335:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772604.073343:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772604.073345:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772604.073359:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:3.0:1790772604.073360:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772604.073375:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534488448:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772604.073384:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534488960/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772604.073398:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772604.073403:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772604.073405:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772604.073412:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1599 00000100:00080000:0.0:1790772604.073418:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772604.073422:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534488576:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772604.073431:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772604.073440:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772604.073458:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534489088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772604.073474:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772604.073483:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:3.0:1790772604.073487:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772604.073489:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772604.073503:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1599 00000100:00080000:0.0:1790772604.073510:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772604.073520:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772604/real 1790772604] req@ffff9c4ec7a7c700 x1877759534488576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772604.073521:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772604.073536:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772604.073560:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534488576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772604.073568:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534489216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772604.073602:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772604.073603:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772604.073615:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772604.073619:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772604.073621:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772604.073632:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534488576:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772604.073635:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1599 00000100:00100000:3.0:1790772604.073690:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534488704:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772604.073701:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772604.073713:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772604.073723:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:3.0:1790772604.073730:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772604.073751:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534489344/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772604.073756:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772604/real 1790772604] req@ffff9c4ec7a7fb80 x1877759534488704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772604.073824:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772604.073832:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534488704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772604.073835:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772604.073844:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772604.073862:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1599 00000100:00080000:3.0:1790772604.073866:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772604.073870:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772604.073879:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772604.073886:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772604.073893:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534488704:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790772604.073899:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772604.073910:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534488832:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772604.073926:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534489472/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772604.073949:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772604.073967:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772604/real 1790772604] req@ffff9c4ec7a7e300 x1877759534488832/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772604.073985:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534488832/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772604.073999:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772604.074004:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772604.074009:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534488832:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772604.074017:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534488960:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772604.074038:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772604.074050:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772604/real 1790772604] req@ffff9c4ec7a7ca80 x1877759534488960/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772604.074065:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534488960/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772604.074078:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772604.074082:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772604.074086:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534488960:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772604.074109:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534489088:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772604.074131:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772604.074144:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772604/real 1790772604] req@ffff9c4ec7a7ed80 x1877759534489088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772604.074161:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534489088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772604.074182:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772604.074186:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772604.074190:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534489088:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772604.074198:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534489216:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772604.074217:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772604.074230:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772604/real 1790772604] req@ffff9c4ec7a7d880 x1877759534489216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772604.074245:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534489216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772604.074256:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772604.074259:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772604.074264:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534489216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772604.074271:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534489344:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772604.074291:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772604.074302:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772604/real 1790772604] req@ffff9c4ec7a7c380 x1877759534489344/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772604.074317:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534489344/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772604.074341:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772604.074345:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772604.074350:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534489344:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772604.074357:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534489472:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772604.074376:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772604.074388:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772604/real 1790772604] req@ffff9c4ec7a7f480 x1877759534489472/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772604.074402:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534489472/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772604.074415:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772604.074420:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772604.074424:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534489472:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772604.136833:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772604.136836:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772604.136842:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772604.136843:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772604.200847:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772604.200852:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772604.200969:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772604.328743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772604.328764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772604.392734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772604.456920:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772604.520809:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772604.520824:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772604.520846:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772604.584745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772604.648758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772604.648818:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772604.648828:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772604.776795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772604.776806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772604.776844:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772604.776860:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772604.840786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772604.840792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772604.840838:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772604.840841:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772604.840874:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772604.840885:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772604.968787:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772605.032791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772605.096807:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772605.160809:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772605.160810:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772605.160810:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772605.160836:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772605.224776:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772605.224845:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772605.225007:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772605.352815:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772605.352822:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772605.416838:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772605.480778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772605.544808:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772605.544817:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772605.544818:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772605.608779:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772605.672786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772605.672802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772605.672807:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772605.800783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772605.800798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772605.800826:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772605.800832:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772605.864775:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772605.864808:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772605.864848:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772605.864881:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772605.864881:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772605.864886:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772605.992790:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772606.056778:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772606.121370:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772606.184814:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772606.184903:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772606.184913:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772606.184914:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772606.248784:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772606.248849:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772606.248973:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772606.376809:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772606.376809:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772606.440779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772606.504820:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772606.568810:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772606.568810:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772606.568846:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772606.632769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772606.696844:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772606.696875:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772606.696881:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772606.824823:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772606.824823:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772606.824851:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772606.824957:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772606.888795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772606.888838:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772606.888859:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772606.888859:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772606.888885:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772606.888902:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772607.016788:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772607.080822:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772607.144798:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772607.208808:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772607.208818:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772607.208818:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772607.208843:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772607.272784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772607.272861:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772607.272894:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772607.400789:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772607.400803:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772607.464800:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772607.528829:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772607.592809:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772607.592811:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772607.592829:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772607.656787:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772607.720783:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772607.720795:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772607.720815:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772607.848788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772607.848813:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772607.848832:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772607.848872:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772607.912816:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772607.912840:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772607.912842:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772607.912870:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772607.912892:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772607.912900:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772608.040777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772608.104791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772608.168765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772608.232773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772608.232802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772608.232808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772608.232830:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772608.296810:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772608.296856:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772608.296979:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772608.424792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772608.424806:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772608.488795:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772608.552771:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772608.616762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772608.616808:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772608.616817:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772608.680800:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772608.744773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772608.744798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772608.744832:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772608.872769:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772608.872801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772608.872814:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772608.872820:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772608.936794:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772608.936801:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772608.936827:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772608.936865:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772608.936892:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772608.936904:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772609.064802:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772609.128772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772609.192783:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772609.192814:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772609.192825:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772609.192833:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772609.192849:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1605 00000100:00080000:0.0:1790772609.192866:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772609.192887:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772609.192963:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534489600/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772609.193019:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772609.193025:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772609.193028:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772609.193035:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1605 00000100:00080000:0.0:1790772609.193043:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772609.193052:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772609.193071:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772609.193073:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534489600:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772609.193103:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534489728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772609.193121:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772609.193127:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772609.193130:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772609.193137:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1605 00000100:00080000:0.0:1790772609.193144:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772609.193152:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772609.193160:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772609.193183:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534489856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772609.193186:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772609.193212:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772609.193222:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772609.193227:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772609.193240:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1605 00000100:00080000:0.0:1790772609.193250:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772609.193258:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772609/real 1790772609] req@ffff9c4ec7a7df80 x1877759534489600/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772609.193272:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772609.193289:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772609.193296:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534489600/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772609.193324:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790772609.193335:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534489984/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772609.193336:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772609.193352:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534489600:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772609.193385:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772609.193394:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772609.193399:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772609.193412:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1605 00000100:00100000:3.0:1790772609.193416:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534489728:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772609.193426:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772609.193440:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772609.193451:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:3.0:1790772609.193470:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772609.193479:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534490112/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772609.193496:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772609/real 1790772609] req@ffff9c4ec7a7f480 x1877759534489728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772609.193499:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772609.193517:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772609.193524:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772609.193536:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1605 00000100:00100000:3.0:1790772609.193539:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534489728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772609.193543:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772609.193558:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772609.193569:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1790772609.193593:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790772609.193600:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534490240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772609.193609:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772609.193625:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534489728:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772609.193645:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772609.193707:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534489856:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772609.193733:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772609.193738:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772609.193748:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1605 00000800:00000100:3.0:1790772609.193755:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772609.193759:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772609.193775:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772609.193785:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772609/real 1790772609] req@ffff9c4ec7a7c380 x1877759534489856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790772609.193787:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772609.193815:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534489856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772609.193821:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534490368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772609.193864:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772609.193865:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772609.193880:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772609.193886:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772609.193887:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772609.193909:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1605 00000100:00080000:0.0:1790772609.193920:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772609.193925:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534489856:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772609.193937:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772609.193948:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534489984:192.168.204.150@tcp:8:kworker.0 00000008:00100000:0.0:1790772609.193956:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772609.193993:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534490496/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772609.194002:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772609.194022:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772609.194029:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772609/real 1790772609] req@ffff9c4ec7a7d880 x1877759534489984/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772609.194031:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772609.194038:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772609.194056:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1605 00000100:00080000:0.0:1790772609.194064:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772609.194066:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534489984/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772609.194077:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772609.194093:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790772609.194094:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:3.0:1790772609.194108:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772609.194114:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534489984:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772609.194123:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534490624/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772609.194145:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534490112:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772609.194177:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772609.194191:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772609/real 1790772609] req@ffff9c4ec7a7ed80 x1877759534490112/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772609.194207:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534490112/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772609.194230:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772609.194235:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772609.194240:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534490112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772609.194247:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534490240:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772609.194268:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772609.194280:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772609/real 1790772609] req@ffff9c4ec7a7ca80 x1877759534490240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772609.194298:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534490240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772609.194329:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772609.194337:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772609.194345:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534490240:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772609.194358:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534490368:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772609.194399:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772609.194421:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772609/real 1790772609] req@ffff9c4ec7a7e300 x1877759534490368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772609.194438:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534490368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772609.194450:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772609.194455:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772609.194461:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534490368:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772609.194469:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534490496:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772609.194496:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772609.194508:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772609/real 1790772609] req@ffff9c4ec7a7fb80 x1877759534490496/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772609.194524:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534490496/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772609.194538:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772609.194542:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772609.194547:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534490496:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772609.194576:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534490624:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772609.194617:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772609.194640:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772609/real 1790772609] req@ffff9c4ec7a7c700 x1877759534490624/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772609.194727:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534490624/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772609.194743:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772609.194749:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772609.194754:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534490624:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772609.256801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772609.256810:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772609.256813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772609.256831:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772609.320801:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772609.320819:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772609.320904:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772609.448788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772609.448799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772609.512787:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772609.576796:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772609.640791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772609.640813:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772609.640834:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772609.704817:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772609.768764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772609.768842:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772609.768847:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772609.896759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772609.896759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772609.896759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772609.896794:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772609.960752:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772609.960752:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772609.960752:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772609.960763:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772609.960790:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772609.960808:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772610.088842:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772610.152727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772610.216713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772610.280750:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772610.280766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772610.280770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772610.280774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772610.344749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772610.344809:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772610.344876:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772610.472808:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772610.472852:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772610.536783:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772610.600822:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772610.664741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772610.664743:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772610.664770:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772610.728836:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772610.792771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772610.792855:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772610.792870:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772610.920780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772610.920857:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772610.920861:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772610.920863:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772610.984790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772610.984837:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772610.984867:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772610.984877:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772610.984897:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772610.984924:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772611.112751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772611.176742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772611.240746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772611.304806:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772611.304853:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772611.304853:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772611.304864:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772611.368840:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772611.368863:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772611.368977:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772611.496773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772611.496808:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772611.560773:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772611.624907:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772611.688793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772611.688831:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772611.688835:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772611.752774:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772611.816806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772611.816814:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772611.816847:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772611.944773:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772611.944773:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772611.944784:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772611.944813:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772612.008765:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772612.008810:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772612.008812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772612.008835:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772612.008874:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772612.008887:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772612.136772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772612.200750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772612.264778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772612.328761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772612.328808:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772612.328812:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772612.328834:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772612.392818:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772612.392843:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772612.392925:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772612.520758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772612.520763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772612.584752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772612.648763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772612.712751:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772612.712782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772612.712797:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772612.776800:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772612.840779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772612.840794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772612.840798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772612.968766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772612.968779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772612.968802:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772612.968806:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772613.032773:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772613.032803:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772613.032810:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772613.032835:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772613.032854:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772613.032871:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772613.160762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772613.224754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772613.288747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772613.352760:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772613.352806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772613.352814:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772613.352823:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772613.416772:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772613.416832:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772613.416949:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772613.544752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772613.544779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772613.608763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772613.672782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772613.736796:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772613.736858:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772613.736862:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772613.800757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772613.864815:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772613.864836:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772613.864854:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772613.992788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772613.992801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772613.992834:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772613.992834:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772614.056770:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772614.056822:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772614.056824:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772614.056824:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772614.056875:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772614.056900:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772614.184768:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772614.248761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772614.312779:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772614.312811:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772614.312822:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772614.312830:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772614.312847:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1610 00000100:00080000:2.0:1790772614.312866:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772614.312885:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772614.312953:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534490752/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772614.312999:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772614.313005:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772614.313008:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772614.313014:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1610 00000100:00080000:2.0:1790772614.313019:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772614.313027:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772614.313043:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772614.313062:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534490880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772614.313075:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534490752:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772614.313085:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772614.313092:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772614.313094:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772614.313101:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1610 00000100:00080000:2.0:1790772614.313106:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772614.313112:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772614.313119:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772614.313135:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534491008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772614.313148:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772614.313155:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772614.313158:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772614.313164:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1610 00000100:00080000:2.0:1790772614.313172:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772614.313179:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772614.313187:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772614.313198:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772614.313206:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534491136/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772614.313223:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772614.313229:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772614.313232:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772614.313238:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1610 00000100:00080000:2.0:1790772614.313242:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772614.313249:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772614.313256:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772614.313274:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772614/real 1790772614] req@ffff9c4efbcc1500 x1877759534490752/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772614.313277:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534491264/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772614.313342:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772614.313348:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772614.313352:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772614.313357:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534490752/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772614.313364:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1610 00000100:00080000:3.0:1790772614.313385:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772614.313392:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772614.313397:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772614.313409:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772614.313411:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534490752:192.168.204.150@tcp:250:kworker.0 00000008:00100000:2.0:1790772614.313428:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772614.313448:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534491392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772614.313465:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772614.313468:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534490880:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772614.313481:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772614.313489:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772614.313502:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1610 00000100:00080000:2.0:1790772614.313513:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1790772614.313514:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772614.313535:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772614.313544:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772614/real 1790772614] req@ffff9c4efbcc2a00 x1877759534490880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790772614.313553:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772614.313628:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534490880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772614.313629:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534491520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772614.313707:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772614.313719:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772614.313732:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534490880:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772614.313741:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772614.313749:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534491008:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772614.313751:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772614.313761:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772614.313772:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1610 00000100:00080000:2.0:1790772614.313781:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772614.313796:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:3.0:1790772614.313806:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790772614.313813:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772614.313830:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772614/real 1790772614] req@ffff9c4efbcc1c00 x1877759534491008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772614.313843:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534491648/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772614.313861:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534491008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772614.313868:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772614.313884:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772614.313892:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772614.313897:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772614.313903:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1610 00000100:00080000:3.0:1790772614.313912:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772614.313913:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772614.313925:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534491008:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772614.313931:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772614.313941:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534491136:192.168.204.150@tcp:8:kworker.0 00000008:00100000:2.0:1790772614.313947:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772614.313983:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534491776/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772614.313996:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772614.314033:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772614/real 1790772614] req@ffff9c4efbcc1880 x1877759534491136/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772614.314061:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534491136/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772614.314095:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772614.314100:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772614.314106:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534491136:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772614.314115:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534491264:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772614.314147:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772614.314163:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772614/real 1790772614] req@ffff9c4efbcc2300 x1877759534491264/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772614.314178:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534491264/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772614.314192:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772614.314197:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772614.314201:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534491264:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772614.314230:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534491392:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772614.314251:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772614.314265:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772614/real 1790772614] req@ffff9c4efbcc2680 x1877759534491392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772614.314280:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534491392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772614.314300:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772614.314304:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772614.314310:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534491392:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772614.314318:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534491520:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772614.314338:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772614.314350:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772614/real 1790772614] req@ffff9c4efbcc0700 x1877759534491520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772614.314364:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534491520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772614.314375:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772614.314379:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772614.314384:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534491520:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772614.314390:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534491648:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772614.314409:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772614.314420:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772614/real 1790772614] req@ffff9c4efbcc1f80 x1877759534491648/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772614.314435:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534491648/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772614.314448:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772614.314452:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772614.314457:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534491648:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772614.314463:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534491776:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772614.314482:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772614.314495:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772614/real 1790772614] req@ffff9c4efbcc0a80 x1877759534491776/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772614.314510:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534491776/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772614.314522:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772614.314526:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772614.314530:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534491776:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772614.376795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772614.376819:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772614.376849:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772614.376861:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772614.440793:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772614.440878:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772614.440972:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772614.568768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772614.568779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772614.632763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772614.696795:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772614.760770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772614.760829:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772614.760835:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772614.824802:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772614.888771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772614.888779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772614.888816:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772615.016827:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772615.016846:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772615.016870:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772615.017211:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772615.080792:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772615.080835:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772615.080835:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772615.080859:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772615.080876:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772615.080909:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772615.208755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772615.272729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772615.336753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772615.400794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772615.400822:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772615.400829:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772615.400880:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772615.464839:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772615.464847:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772615.464994:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772615.592811:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772615.592812:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772615.656786:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772615.720786:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772615.784773:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772615.784803:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772615.784842:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772615.848767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772615.912808:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772615.912832:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772615.912833:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772616.040824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772616.040886:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772616.040970:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772616.041058:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772616.104798:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772616.104852:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772616.104884:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772616.104917:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772616.104919:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772616.104924:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772616.232777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772616.296769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772616.360733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772616.424829:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772616.424829:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772616.424857:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772616.424893:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772616.488782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772616.488883:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772616.489004:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772616.616759:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772616.616770:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772616.680738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772616.744779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772616.808783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772616.808784:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772616.808796:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772616.872764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772616.936849:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772616.936854:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772616.936859:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772617.064897:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772617.064902:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772617.064914:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772617.064946:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772617.128785:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772617.128827:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772617.128835:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772617.128835:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772617.128863:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772617.128900:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772617.256760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772617.320762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772617.384785:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772617.448824:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772617.448825:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772617.448827:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772617.448845:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772617.512839:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772617.512845:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772617.513053:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772617.640766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772617.640778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772617.704772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772617.768767:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772617.832778:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772617.832816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772617.832826:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772617.896763:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772617.960805:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772617.960873:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772617.960876:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772618.088838:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772618.088938:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772618.089014:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772618.089056:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772618.152789:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772618.152795:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772618.152812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772618.152826:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772618.152875:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772618.152906:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772618.280775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772618.344757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772618.408755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772618.472778:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772618.472798:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772618.472803:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772618.472846:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772618.536782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772618.536852:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772618.537068:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772618.664758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772618.664766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772618.728761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772618.792881:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772618.856767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772618.856773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772618.856837:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772618.920775:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772618.984763:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772618.984791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772618.984825:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772619.112824:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772619.112875:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772619.112941:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772619.113051:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772619.176777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772619.176783:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772619.176823:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772619.176856:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772619.176878:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772619.176887:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772619.304750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772619.368753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772619.432775:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772619.432849:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772619.432860:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772619.432867:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772619.432882:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1615 00000100:00080000:0.0:1790772619.432900:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772619.432919:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772619.432990:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534491904/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772619.433047:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772619.433055:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772619.433061:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772619.433071:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1615 00000100:00080000:0.0:1790772619.433082:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772619.433100:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534491904:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772619.433101:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772619.433143:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772619.433171:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534492032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772619.433190:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772619.433195:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772619.433198:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772619.433206:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1615 00000100:00080000:0.0:1790772619.433217:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772619.433225:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772619.433232:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1790772619.433243:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772619.433251:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534492160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772619.433266:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772619.433272:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772619.433275:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772619.433282:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1615 00000100:00080000:0.0:1790772619.433288:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772619.433293:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772619.433300:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772619.433318:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534492288/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772619.433332:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772619/real 1790772619] req@ffff9c4ec7a7d180 x1877759534491904/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772619.433336:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772619.433361:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772619.433366:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772619.433374:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534491904/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772619.433376:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1615 00000100:00080000:0.0:1790772619.433387:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772619.433401:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772619.433412:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790772619.433413:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:3.0:1790772619.433428:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772619.433440:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534492416/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772619.433441:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534491904:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772619.433466:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772619.433471:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772619.433473:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772619.433511:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1615 00000100:00080000:0.0:1790772619.433517:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772619.433526:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772619.433536:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772619.433536:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534492032:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772619.433569:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534492544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772619.433586:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772619.433596:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:3.0:1790772619.433597:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772619.433600:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772619.433611:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1615 00000100:00080000:0.0:1790772619.433619:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772619.433626:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772619/real 1790772619] req@ffff9c4ec7a7c700 x1877759534492032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772619.433628:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772619.433641:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772619.433696:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534492032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772619.433723:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534492672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772619.433742:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772619.433743:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772619.433757:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772619.433761:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772619.433764:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772619.433773:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534492032:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772619.433777:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1615 00000100:00080000:0.0:1790772619.433788:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772619.433792:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534492160:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772619.433801:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772619.433812:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772619.433830:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534492800/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772619.433837:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772619.433855:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772619.433863:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1790772619.433863:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772619/real 1790772619] req@ffff9c4ec7a7fb80 x1877759534492160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772619.433870:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772619.433884:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1615 00000100:00080000:0.0:1790772619.433895:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772619.433902:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534492160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772619.433905:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772619.433918:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:3.0:1790772619.433930:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772619.433941:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772619.433944:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534492928/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772619.433952:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534492160:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772619.433975:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534492288:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772619.434005:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772619.434020:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772619/real 1790772619] req@ffff9c4ec7a7e300 x1877759534492288/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772619.434035:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534492288/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772619.434049:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772619.434054:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772619.434059:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534492288:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772619.434066:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534492416:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772619.434085:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772619.434097:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772619/real 1790772619] req@ffff9c4ec7a7ca80 x1877759534492416/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772619.434111:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534492416/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772619.434127:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772619.434133:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772619.434141:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534492416:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772619.434174:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534492544:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772619.434217:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772619.434240:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772619/real 1790772619] req@ffff9c4ec7a7ed80 x1877759534492544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772619.434259:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534492544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772619.434281:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772619.434286:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772619.434290:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534492544:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772619.434299:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534492672:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772619.434325:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772619.434339:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772619/real 1790772619] req@ffff9c4ec7a7d880 x1877759534492672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772619.434354:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534492672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772619.434365:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772619.434369:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772619.434373:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534492672:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772619.434380:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534492800:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772619.434399:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772619.434410:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772619/real 1790772619] req@ffff9c4ec7a7c380 x1877759534492800/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772619.434426:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534492800/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772619.434449:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772619.434453:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772619.434458:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534492800:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772619.434465:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534492928:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772619.434516:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772619.434531:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772619/real 1790772619] req@ffff9c4ec7a7f480 x1877759534492928/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772619.434547:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534492928/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772619.434561:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772619.434565:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772619.434570:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534492928:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772619.496797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772619.496799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772619.496799:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772619.496825:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772619.560817:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772619.560853:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772619.561070:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772619.688766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772619.688766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772619.752756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772619.816780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772619.880818:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772619.880827:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772619.880827:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772619.944773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772620.008878:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772620.008885:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772620.008888:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772620.136808:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772620.136808:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772620.136987:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772620.137757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772620.200767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772620.200780:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772620.200802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772620.200806:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772620.200846:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772620.200846:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772620.328742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772620.392751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772620.456753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772620.520773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772620.520783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772620.520788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772620.520816:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772620.584792:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772620.584814:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772620.584907:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772620.712761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772620.712776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772620.776776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772620.840741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772620.904758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772620.904780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772620.904814:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772620.968788:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772621.032756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772621.032797:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772621.032806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772621.160767:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772621.160782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772621.160790:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772621.160805:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772621.224754:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772621.224792:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772621.224817:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772621.224820:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772621.224831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772621.224861:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772621.352739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772621.416754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772621.480757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772621.544781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772621.544781:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772621.544787:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772621.544809:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772621.608768:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772621.608821:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772621.608883:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772621.736773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772621.736808:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772621.800756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772621.864798:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772621.928739:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772621.928783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772621.928788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772621.992758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772622.056770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772622.056788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772622.056814:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772622.184771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772622.184799:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772622.184799:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772622.184814:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772622.248777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772622.248795:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772622.248815:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772622.248838:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772622.248859:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772622.248887:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772622.376758:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772622.440753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772622.504736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772622.568766:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772622.568790:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772622.568797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772622.568798:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772622.632770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772622.632811:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772622.633010:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772622.760738:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772622.760770:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772622.824750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772622.888777:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772622.952767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772622.952787:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772622.952794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772623.016758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772623.080750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772623.080775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772623.080784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772623.208777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772623.208835:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772623.208842:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772623.208855:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772623.272793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772623.272869:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772623.272876:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772623.272878:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772623.272907:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772623.272931:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772623.400756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772623.464766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772623.528779:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772623.592778:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772623.592861:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772623.592883:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772623.592883:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772623.656799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772623.656872:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772623.657018:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772623.784766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772623.784784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772623.848780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772623.912796:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772623.976777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772623.976783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772623.976796:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772624.040768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772624.104771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772624.104783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772624.104790:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772624.232781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772624.232785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772624.232883:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772624.232886:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772624.296791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772624.296794:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772624.296855:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772624.296868:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772624.296876:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772624.296908:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772624.424754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772624.488766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772624.552748:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772624.552760:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772624.552767:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772624.552776:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772624.552783:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1620 00000100:00080000:0.0:1790772624.552803:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772624.552822:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772624.552880:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534493056/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772624.552923:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772624.552929:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772624.552932:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772624.552938:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1620 00000100:00080000:0.0:1790772624.552944:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772624.552952:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772624.552968:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772624.552988:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534493184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772624.552995:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534493056:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772624.553023:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772624.553028:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772624.553031:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772624.553039:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1620 00000100:00080000:0.0:1790772624.553044:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772624.553049:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772624.553057:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772624.553074:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534493312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772624.553088:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772624.553093:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772624.553097:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772624.553102:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1620 00000100:00080000:0.0:1790772624.553107:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1790772624.553115:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772624.553117:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772624.553129:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772624.553145:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534493440/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772624.553168:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772624.553174:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772624.553177:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772624.553187:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1620 00000100:00100000:3.0:1790772624.553189:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772624/real 1790772624] req@ffff9c4ec7a7df80 x1877759534493056/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772624.553193:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772624.553206:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772624.553220:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772624.553245:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534493056/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772624.553246:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534493568/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772624.553314:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772624.553318:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772624.553320:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772624.553328:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772624.553335:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772624.553342:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1620 00000100:00100000:3.0:1790772624.553350:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534493056:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772624.553352:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772624.553368:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772624.553404:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772624.553425:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534493184:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772624.553436:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534493696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772624.553466:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1790772624.553475:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772624.553482:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772624.553491:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772624.553502:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772624/real 1790772624] req@ffff9c4ec7a7f480 x1877759534493184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772624.553504:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1620 00000100:00080000:0.0:1790772624.553520:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772624.553537:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772624.553544:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534493184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790772624.553565:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1790772624.553572:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772624.553584:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772624.553590:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534493184:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772624.553600:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534493824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772624.553606:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534493312:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772624.553633:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772624.553640:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772624.553644:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1790772624.553683:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772624.553703:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1620 00000100:00100000:3.0:1790772624.553707:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772624/real 1790772624] req@ffff9c4ec7a7c380 x1877759534493312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772624.553711:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772624.553728:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772624.553738:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534493312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790772624.553739:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:3.0:1790772624.553764:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790772624.553770:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534493952/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772624.553777:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772624.553791:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534493312:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772624.553800:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772624.553810:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534493440:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790772624.553811:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772624.553818:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772624.553834:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1620 00000100:00080000:0.0:1790772624.553840:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772624.553847:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772624.553854:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772624.553871:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534494080/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772624.553886:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772624.553924:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772624/real 1790772624] req@ffff9c4ec7a7d880 x1877759534493440/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772624.553947:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534493440/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772624.553962:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772624.553967:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772624.553972:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534493440:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772624.553981:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534493568:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772624.554011:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772624.554025:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772624/real 1790772624] req@ffff9c4ec7a7ed80 x1877759534493568/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772624.554050:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534493568/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772624.554063:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772624.554067:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772624.554071:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534493568:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772624.554095:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534493696:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772624.554118:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772624.554131:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772624/real 1790772624] req@ffff9c4ec7a7ca80 x1877759534493696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772624.554147:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534493696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772624.554166:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772624.554171:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772624.554175:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534493696:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772624.554183:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534493824:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772624.554202:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772624.554214:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772624/real 1790772624] req@ffff9c4ec7a7e300 x1877759534493824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772624.554229:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534493824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772624.554249:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772624.554253:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772624.554258:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534493824:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772624.554265:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534493952:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772624.554284:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772624.554295:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772624/real 1790772624] req@ffff9c4ec7a7fb80 x1877759534493952/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772624.554310:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534493952/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772624.554324:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772624.554328:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772624.554332:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534493952:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772624.554339:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534494080:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772624.554358:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772624.554370:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772624/real 1790772624] req@ffff9c4ec7a7c700 x1877759534494080/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772624.554415:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534494080/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772624.554429:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772624.554433:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772624.554439:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534494080:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772624.616787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772624.616842:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772624.616855:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772624.616872:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772624.680753:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772624.680809:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772624.680958:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772624.808733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772624.808784:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772624.872747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772624.936788:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772625.000768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772625.000778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772625.000802:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772625.064798:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772625.128793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772625.128842:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772625.128844:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772625.256780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772625.256792:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772625.256827:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772625.256831:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772625.320772:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772625.320776:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772625.320800:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772625.320810:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772625.320864:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772625.320865:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772625.448767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772625.512744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772625.576751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772625.640771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772625.640773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772625.640780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772625.640807:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772625.704779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772625.704853:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772625.704920:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772625.832755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772625.832788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772625.896758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772625.960902:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772626.024769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772626.024786:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772626.024813:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772626.088767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772626.152755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772626.152776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772626.152788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772626.280771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772626.280771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772626.280791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772626.280809:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772626.344805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772626.344806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772626.344806:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772626.344830:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772626.344863:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772626.344890:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772626.472766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772626.536737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772626.600759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772626.664753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772626.664784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772626.664788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772626.664802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772626.728819:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772626.728877:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772626.728891:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772626.856766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772626.856775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772626.920749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772626.984780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772627.048734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772627.048761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772627.048780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772627.112782:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772627.176746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772627.176773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772627.176793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772627.304781:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772627.304782:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772627.304792:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772627.304792:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772627.368790:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772627.368795:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772627.368833:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772627.368833:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772627.368862:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772627.368878:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772627.496752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772627.560750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772627.624726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772627.688758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772627.688775:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772627.688791:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772627.688801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772627.752805:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772627.752861:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772627.752907:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772627.880780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772627.880783:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772627.944747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772628.008777:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772628.072743:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772628.072778:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772628.072783:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772628.136756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772628.200750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772628.200774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772628.200777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772628.328789:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772628.328790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772628.328805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772628.328832:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772628.392780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772628.392786:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772628.392825:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772628.392826:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772628.392865:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772628.392870:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772628.520757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772628.584736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772628.648746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772628.712765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772628.712802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772628.712803:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772628.712813:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772628.776794:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772628.776884:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772628.776887:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772628.904754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772628.904778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772628.968745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772629.032799:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772629.096767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772629.096778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772629.096795:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772629.160739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772629.224763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772629.224771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772629.224779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772629.352765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772629.352786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772629.352795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772629.352798:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772629.416772:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772629.416785:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772629.416805:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772629.416807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772629.416838:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772629.416840:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772629.544749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772629.608759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772629.672777:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772629.672840:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772629.672851:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772629.672858:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772629.672874:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1625 00000100:00080000:2.0:1790772629.672891:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772629.672911:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772629.672989:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534494208/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772629.673046:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772629.673052:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772629.673055:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772629.673063:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1625 00000100:00080000:2.0:1790772629.673070:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772629.673080:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772629.673101:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772629.673108:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534494208:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772629.673129:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534494336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772629.673148:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772629.673155:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772629.673160:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772629.673171:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1625 00000100:00080000:2.0:1790772629.673204:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772629.673218:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:3.0:1790772629.673227:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790772629.673236:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772629.673269:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534494464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772629.673309:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772629.673315:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772629.673318:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772629.673327:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1625 00000100:00080000:2.0:1790772629.673335:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772629.673340:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772629/real 1790772629] req@ffff9c4efbcc3100 x1877759534494208/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772629.673351:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772629.673369:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772629.673389:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534494208/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772629.673396:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534494592/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772629.673438:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772629.673449:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772629.673459:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772629.673464:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1790772629.673468:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534494208:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772629.673474:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772629.673491:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1625 00000100:00080000:2.0:1790772629.673501:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772629.673514:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772629.673518:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534494336:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790772629.673535:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772629.673572:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534494720/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772629.673574:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772629.673600:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772629/real 1790772629] req@ffff9c4efbcc0380 x1877759534494336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772629.673607:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772629.673633:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772629.673640:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772629.673643:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534494336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772629.673713:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1625 00000100:00080000:3.0:1790772629.673716:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772629.673721:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772629.673728:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772629.673741:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772629.673743:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534494336:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790772629.673759:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772629.673763:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534494464:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772629.673791:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534494848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772629.673807:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772629.673813:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772629.673824:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1790772629.673830:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772629/real 1790772629] req@ffff9c4efbcc0a80 x1877759534494464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772629.673833:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772629.673855:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1625 00000100:00100000:3.0:1790772629.673865:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534494464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772629.673869:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772629.673888:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772629.673891:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772629.673898:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772629.673909:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534494464:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790772629.673911:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772629.673937:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534494592:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772629.673949:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534494976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772629.673975:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772629.673983:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772629.673988:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1790772629.673999:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772629.674004:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1625 00000100:00080000:2.0:1790772629.674018:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772629.674031:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772629.674039:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772629/real 1790772629] req@ffff9c4efbcc1f80 x1877759534494592/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790772629.674047:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772629.674082:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534495104/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772629.674089:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534494592/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772629.674126:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772629.674134:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772629.674134:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772629.674143:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772629.674148:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772629.674159:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1625 00000100:00100000:3.0:1790772629.674161:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534494592:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772629.674179:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772629.674198:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772629.674208:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534494720:192.168.204.150@tcp:8:kworker.0 00000008:00100000:2.0:1790772629.674211:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772629.674251:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534495232/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772629.674260:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772629.674312:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772629/real 1790772629] req@ffff9c4efbcc0700 x1877759534494720/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772629.674338:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534494720/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772629.674353:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772629.674358:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772629.674363:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534494720:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772629.674372:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534494848:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772629.674403:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772629.674418:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772629/real 1790772629] req@ffff9c4efbcc2680 x1877759534494848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772629.674433:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534494848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772629.674463:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772629.674468:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772629.674472:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534494848:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772629.674480:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534494976:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772629.674499:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772629.674512:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772629/real 1790772629] req@ffff9c4efbcc2300 x1877759534494976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772629.674525:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534494976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772629.674537:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772629.674556:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772629.674561:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534494976:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772629.674575:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534495104:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772629.674599:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772629.674612:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772629/real 1790772629] req@ffff9c4efbcc1880 x1877759534495104/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772629.674628:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534495104/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772629.674642:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772629.674646:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772629.674696:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534495104:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772629.674725:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534495232:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772629.674758:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772629.674773:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772629/real 1790772629] req@ffff9c4efbcc1c00 x1877759534495232/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772629.674789:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534495232/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772629.674805:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772629.674811:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772629.674816:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534495232:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772629.736772:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772629.736773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772629.736787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772629.736813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772629.800783:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772629.800851:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772629.800915:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772629.928764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772629.928785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772629.992764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772630.056749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772630.120786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772630.120786:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772630.120809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772630.184761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772630.248745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772630.248792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772630.248792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772630.376764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772630.376798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772630.376804:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772630.376804:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772630.440771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772630.440798:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772630.440804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772630.440836:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772630.440855:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772630.440861:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772630.568755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772630.632751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772630.696782:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772630.760772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772630.760776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772630.760784:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772630.760790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772630.824750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772630.824803:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772630.824895:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772630.952772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772630.952776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772631.016745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772631.080773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772631.144762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772631.144775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772631.144782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772631.208768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772631.272776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772631.272789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772631.272792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772631.400772:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772631.400777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772631.400792:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772631.400809:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772631.464787:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772631.464787:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772631.464789:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772631.464827:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772631.464876:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772631.464890:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772631.592742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772631.656760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772631.720774:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772631.784810:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772631.784829:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772631.784832:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772631.784835:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772631.848852:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772631.848858:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772631.849051:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772631.976778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772631.976791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772632.040749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772632.104756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772632.168796:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772632.168802:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772632.168806:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772632.232735:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772632.296769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772632.296784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772632.296792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772632.424789:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772632.424800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772632.424868:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772632.424874:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772632.488778:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772632.488813:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772632.488825:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772632.488858:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772632.488865:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772632.488881:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772632.616750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772632.680753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772632.744761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772632.808766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772632.808794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772632.808802:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772632.808805:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772632.872767:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772632.872839:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772632.872903:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772633.000775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772633.000780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772633.064776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772633.128852:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772633.192772:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772633.192786:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772633.192799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772633.256765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772633.320759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772633.320769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772633.320800:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772633.448776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772633.448777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772633.448796:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772633.448801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772633.512763:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772633.512799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772633.512802:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772633.512815:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772633.512842:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772633.512873:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772633.640810:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772633.704784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772633.768765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772633.832755:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772633.832769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772633.832790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772633.832801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772633.896773:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772633.896821:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772633.896930:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772634.024788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772634.024795:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772634.088770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772634.152806:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772634.216775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772634.216777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772634.216795:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772634.280760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772634.344751:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772634.344782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772634.344798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772634.472753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772634.472774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772634.472786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772634.472795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772634.536770:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772634.536783:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772634.536787:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772634.536798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772634.536831:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772634.536839:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772634.664731:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772634.728711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772634.792708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772634.792753:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772634.792759:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772634.792764:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772634.792774:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1630 00000100:00080000:1.0:1790772634.792786:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772634.792799:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772634.792843:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534495360/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772634.792871:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772634.792874:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772634.792875:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772634.792879:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1630 00000100:00080000:1.0:1790772634.792882:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772634.792886:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772634.792897:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772634.792904:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534495360:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772634.792911:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534495488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772634.792920:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772634.792923:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772634.792924:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772634.792928:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1630 00000100:00080000:1.0:1790772634.792930:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772634.792934:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772634.792937:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772634.792946:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534495616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772634.792953:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772634.792956:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772634.792958:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772634.792961:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1630 00000100:00080000:1.0:1790772634.792964:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772634.792976:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772634.792981:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772634.792987:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772634.792991:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534495744/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772634.792999:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772634.793002:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772634.793005:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772634.793009:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1630 00000100:00080000:1.0:1790772634.793011:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772634.793014:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772634.793018:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772634.793026:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534495872/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772634.793034:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772634.793035:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772634/real 1790772634] req@ffff9c4efac4fb80 x1877759534495360/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772634.793042:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772634.793046:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772634.793053:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1630 00000100:00080000:1.0:1790772634.793056:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772634.793060:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772634.793064:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772634.793065:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534495360/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772634.793075:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534496000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772634.793078:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772634.793087:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772634.793110:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534495360:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772634.793116:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772634.793127:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772634.793129:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772634.793134:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1630 00000100:00080000:1.0:1790772634.793138:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772634.793142:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534495488:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772634.793145:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772634.793152:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772634.793165:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534496128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772634.793173:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772634.793174:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772634.793194:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772634.793197:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772634.793219:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1630 00000100:00100000:3.0:1790772634.793221:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772634/real 1790772634] req@ffff9c4efac4df80 x1877759534495488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772634.793223:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772634.793233:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772634.793238:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772634.793238:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534495488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772634.793255:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534496256/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772634.793259:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772634.793267:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772634.793268:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772634.793274:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1790772634.793274:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534495488:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772634.793278:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772634.793285:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1630 00000100:00100000:3.0:1790772634.793285:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534495616:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772634.793292:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772634.793296:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772634.793300:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772634.793309:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534496384/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772634.793313:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772634.793326:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772634/real 1790772634] req@ffff9c4efac4e300 x1877759534495616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772634.793341:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534495616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772634.793348:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772634.793350:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772634.793353:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534495616:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772634.793357:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534495744:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772634.793378:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772634.793390:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772634/real 1790772634] req@ffff9c4efac4f480 x1877759534495744/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772634.793404:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534495744/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772634.793416:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772634.793420:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772634.793424:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534495744:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772634.793431:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534495872:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772634.793449:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772634.793457:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772634/real 1790772634] req@ffff9c4efac4f800 x1877759534495872/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772634.793466:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534495872/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772634.793478:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772634.793481:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772634.793484:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534495872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772634.793488:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534496000:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772634.793500:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772634.793506:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772634/real 1790772634] req@ffff9c4efac4dc00 x1877759534496000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772634.793514:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534496000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772634.793528:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772634.793530:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772634.793532:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534496000:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772634.793554:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534496128:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772634.793567:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772634.793578:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772634/real 1790772634] req@ffff9c4efac4ed80 x1877759534496128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772634.793592:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534496128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772634.793601:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772634.793605:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772634.793609:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534496128:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772634.793617:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534496256:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772634.793636:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772634.793645:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772634/real 1790772634] req@ffff9c4efac4c700 x1877759534496256/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772634.793671:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534496256/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772634.793680:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772634.793682:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772634.793685:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534496256:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772634.793690:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534496384:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772634.793703:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772634.793710:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772634/real 1790772634] req@ffff9c4efac4ea00 x1877759534496384/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772634.793718:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534496384/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772634.793726:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772634.793729:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772634.793731:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534496384:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772634.856727:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772634.856727:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772634.856730:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772634.856730:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772634.920711:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772634.920717:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772634.920772:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772635.048711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772635.048716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772635.112704:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772635.176743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772635.240777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772635.240777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772635.240781:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772635.304730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772635.368739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772635.368744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772635.368747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772635.496763:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772635.496777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772635.496780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772635.496782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772635.560740:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772635.560745:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772635.560801:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772635.560817:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772635.560817:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772635.560825:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772635.688785:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772635.752775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772635.816784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772635.880772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772635.880843:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772635.880855:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772635.880869:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772635.944777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772635.944844:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772635.944966:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772636.072802:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772636.072803:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772636.136750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772636.200780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772636.264798:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772636.264798:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772636.264800:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772636.328739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772636.392768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772636.392776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772636.392825:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772636.520785:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772636.520788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772636.520831:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772636.520842:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772636.584807:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772636.584835:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772636.584853:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772636.584856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772636.584907:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772636.584917:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772636.712813:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772636.776768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772636.840767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772636.904787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772636.904799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772636.904803:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772636.904854:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772636.968783:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772636.968835:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772636.968998:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772637.096762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772637.096769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772637.160762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772637.224780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772637.288755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772637.288778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772637.288805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772637.352739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772637.416760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772637.416800:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772637.416800:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772637.544792:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772637.544805:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772637.544812:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772637.544814:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772637.608793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772637.608799:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772637.608834:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772637.608842:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772637.608880:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772637.608894:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772637.736776:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772637.800762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772637.864767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772637.928735:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772637.928832:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772637.928835:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772637.928836:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772637.992830:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772637.992864:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772637.993015:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772638.120763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772638.120770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772638.184745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772638.248787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772638.312792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772638.312807:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772638.312807:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772638.376781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772638.440772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772638.440793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772638.440804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772638.568791:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772638.568800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772638.568800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772638.568809:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772638.632788:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772638.632788:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772638.632810:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772638.632831:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772638.632891:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772638.632891:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772638.760784:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772638.824771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772638.888779:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772638.952787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772638.952788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772638.952788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772638.952803:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772639.016831:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772639.016852:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772639.016955:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772639.144751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772639.144793:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772639.208777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772639.272759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772639.336767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772639.336799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772639.336821:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772639.400806:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772639.464749:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772639.464785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772639.464820:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772639.592782:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772639.592818:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772639.592872:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772639.592873:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772639.656855:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772639.656856:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772639.656856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772639.656908:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772639.656940:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772639.656956:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772639.784756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772639.848768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772639.912789:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772639.912871:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772639.912882:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772639.912889:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772639.912904:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1635 00000100:00080000:0.0:1790772639.912922:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772639.912942:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772639.913017:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534496512/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772639.913103:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772639.913110:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772639.913116:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772639.913124:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1635 00000100:00080000:0.0:1790772639.913132:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772639.913142:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772639.913160:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772639.913167:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534496512:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772639.913191:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534496640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772639.913211:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772639.913216:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772639.913218:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772639.913225:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1635 00000100:00080000:0.0:1790772639.913230:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772639.913237:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772639.913245:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772639.913260:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534496768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772639.913274:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772639.913284:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1790772639.913284:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772639.913295:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772639.913310:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1635 00000100:00080000:0.0:1790772639.913322:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772639.913334:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772639.913347:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772639.913357:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772639/real 1790772639] req@ffff9c4ec7a7d180 x1877759534496512/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772639.913382:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534496896/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772639.913391:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534496512/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772639.913436:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772639.913437:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772639.913449:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772639.913458:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772639.913459:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772639.913476:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1635 00000100:00100000:3.0:1790772639.913481:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534496512:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772639.913484:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772639.913500:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772639.913508:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772639.913526:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534497024/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772639.913541:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534496640:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772639.913546:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772639.913559:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772639.913563:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772639.913571:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1635 00000100:00080000:0.0:1790772639.913576:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772639.913585:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772639.913592:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1790772639.913603:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772639.913613:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534497152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772639.913628:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772639.913633:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772639/real 1790772639] req@ffff9c4ec7a7c700 x1877759534496640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772639.913638:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772639.913645:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772639.913700:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1635 00000100:00080000:0.0:1790772639.913712:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772639.913720:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534496640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772639.913726:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772639.913741:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772639.913764:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534497280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772639.913765:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772639.913781:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772639.913796:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534496640:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772639.913799:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772639.913810:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772639.913814:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772639.913816:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534496768:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772639.913827:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1635 00000100:00080000:0.0:1790772639.913832:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772639.913840:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772639.913848:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:3.0:1790772639.913858:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772639.913871:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534497408/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772639.913882:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772639/real 1790772639] req@ffff9c4ec7a7fb80 x1877759534496768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772639.913895:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772639.913909:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772639.913916:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772639.913925:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534496768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772639.913926:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1635 00000100:00080000:0.0:1790772639.913936:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772639.913950:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772639.913955:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790772639.913964:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:3.0:1790772639.913965:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772639.913974:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534496768:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772639.913988:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534496896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772639.914001:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534497536/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772639.914044:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772639.914105:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772639/real 1790772639] req@ffff9c4ec7a7e300 x1877759534496896/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772639.914137:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534496896/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772639.914157:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772639.914163:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772639.914169:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534496896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772639.914195:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534497024:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772639.914228:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772639.914243:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772639/real 1790772639] req@ffff9c4ec7a7ca80 x1877759534497024/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772639.914259:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534497024/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772639.914273:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772639.914277:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772639.914282:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534497024:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772639.914290:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534497152:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772639.914311:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772639.914323:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772639/real 1790772639] req@ffff9c4ec7a7ed80 x1877759534497152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772639.914339:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534497152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772639.914359:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772639.914363:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772639.914368:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534497152:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772639.914375:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534497280:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772639.914395:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772639.914408:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772639/real 1790772639] req@ffff9c4ec7a7d880 x1877759534497280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772639.914423:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534497280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772639.914434:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772639.914438:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772639.914442:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534497280:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772639.914451:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534497408:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772639.914489:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772639.914513:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772639/real 1790772639] req@ffff9c4ec7a7c380 x1877759534497408/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772639.914540:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534497408/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772639.914562:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772639.914567:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772639.914572:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534497408:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772639.914581:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534497536:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772639.914612:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772639.914628:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772639/real 1790772639] req@ffff9c4ec7a7f480 x1877759534497536/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772639.914643:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534497536/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772639.914688:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772639.914693:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772639.914698:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534497536:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772639.976766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772639.976788:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772639.976796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772639.976808:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772640.040805:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772640.040827:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772640.040919:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772640.168790:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772640.168801:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772640.232782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772640.296910:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772640.360772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772640.360814:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772640.360817:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772640.424780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772640.488761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772640.488798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772640.488817:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772640.616773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772640.616795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772640.616800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772640.616831:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772640.680790:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772640.680793:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772640.680805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772640.680818:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772640.680855:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772640.680870:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772640.808751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772640.872747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772640.936765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772641.000778:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772641.000790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772641.000822:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772641.000832:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772641.064802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772641.064843:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772641.064887:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772641.192800:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772641.192822:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772641.256743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772641.320814:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772641.384759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772641.384780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772641.384794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772641.448803:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772641.512758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772641.512764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772641.512776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772641.640763:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772641.640793:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772641.640800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772641.640812:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772641.704776:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772641.704785:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772641.704797:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772641.704813:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772641.704824:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772641.704861:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772641.832764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772641.896790:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772641.960812:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772642.024791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772642.024791:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772642.024797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772642.024828:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772642.088795:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772642.088837:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772642.088902:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772642.216796:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772642.216808:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772642.280785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772642.344788:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772642.408769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772642.408791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772642.408799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772642.472888:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772642.536756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772642.536800:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772642.536806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772642.664787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772642.664794:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772642.664814:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772642.664820:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772642.728808:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772642.728864:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772642.728871:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772642.728875:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772642.728922:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772642.728948:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772642.856781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772642.920768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772642.984739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772643.048791:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772643.048803:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772643.048827:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772643.048861:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772643.112793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772643.112827:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772643.113105:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772643.240768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772643.240769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772643.304777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772643.368804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772643.432830:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772643.432875:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772643.432877:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772643.496786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772643.560788:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772643.560790:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772643.560820:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772643.688821:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772643.688826:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772643.688839:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772643.688847:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772643.752804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772643.752809:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772643.752826:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772643.752874:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772643.752892:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772643.752943:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772643.880728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772643.944770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772644.008741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772644.072770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772644.072793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772644.072802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772644.072811:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772644.136773:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772644.136856:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772644.136892:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772644.264754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772644.264770:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772644.328744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772644.392762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772644.456762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772644.456771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772644.456794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772644.520749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772644.584760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772644.584770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772644.584800:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772644.712774:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772644.712807:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772644.712811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772644.712832:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772644.776799:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772644.776813:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772644.776824:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772644.776840:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772644.776861:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772644.776871:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772644.904780:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772644.968750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772645.032781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772645.032782:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772645.032798:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772645.032807:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772645.032823:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1640 00000100:00080000:0.0:1790772645.032841:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772645.032861:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772645.032924:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534497664/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772645.032966:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772645.032972:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772645.032975:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772645.033006:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1640 00000100:00080000:0.0:1790772645.033015:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772645.033028:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772645.033050:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534497664:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790772645.033051:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772645.033095:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534497792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772645.033114:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772645.033120:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772645.033123:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772645.033129:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1640 00000100:00080000:0.0:1790772645.033137:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772645.033145:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772645.033154:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1790772645.033173:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772645.033177:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534497920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772645.033213:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772645.033222:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772645.033226:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772645.033237:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1640 00000100:00080000:0.0:1790772645.033246:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772645.033261:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772645/real 1790772645] req@ffff9c4ec7a7df80 x1877759534497664/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772645.033262:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772645.033282:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772645.033303:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534497664/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772645.033310:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534498048/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772645.033341:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772645.033344:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772645.033354:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772645.033355:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772645.033359:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772645.033373:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534497664:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772645.033374:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1640 00000100:00080000:0.0:1790772645.033384:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772645.033394:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772645.033400:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772645.033418:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534498176/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772645.033423:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534497792:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772645.033458:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772645.033466:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:3.0:1790772645.033473:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772645.033477:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772645.033492:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1640 00000100:00100000:3.0:1790772645.033502:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772645/real 1790772645] req@ffff9c4ec7a7f480 x1877759534497792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772645.033507:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772645.033525:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772645.033534:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534497792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790772645.033537:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772645.033569:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534498304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772645.033574:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772645.033591:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772645.033600:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772645.033603:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534497792:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772645.033612:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772645.033620:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772645.033628:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534497920:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772645.033634:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1640 00000100:00080000:0.0:1790772645.033644:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772645.033688:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772645.033696:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772645.033717:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534498432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772645.033728:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772645.033744:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772645.033750:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772645.033753:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772645.033760:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772645/real 1790772645] req@ffff9c4ec7a7c380 x1877759534497920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772645.033763:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1640 00000100:00080000:0.0:1790772645.033774:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772645.033787:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772645.033797:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772645.033798:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534497920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772645.033818:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790772645.033819:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534498560/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772645.033830:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772645.033845:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534497920:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772645.033870:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534498048:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790772645.033872:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772645.033882:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772645.033888:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772645.033895:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1640 00000100:00080000:0.0:1790772645.033900:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772645.033907:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772645.033915:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772645.033932:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534498688/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772645.033945:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772645.034007:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772645/real 1790772645] req@ffff9c4ec7a7d880 x1877759534498048/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772645.034028:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534498048/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772645.034043:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772645.034048:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772645.034053:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534498048:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772645.034076:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534498176:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772645.034107:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772645.034122:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772645/real 1790772645] req@ffff9c4ec7a7ed80 x1877759534498176/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772645.034138:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534498176/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772645.034152:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772645.034156:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772645.034161:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534498176:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772645.034169:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534498304:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772645.034189:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772645.034201:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772645/real 1790772645] req@ffff9c4ec7a7ca80 x1877759534498304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772645.034216:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534498304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772645.034238:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772645.034243:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772645.034247:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534498304:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772645.034255:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534498432:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772645.034278:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772645.034299:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772645/real 1790772645] req@ffff9c4ec7a7e300 x1877759534498432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772645.034324:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534498432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772645.034343:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772645.034351:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772645.034358:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534498432:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772645.034371:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534498560:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772645.034415:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772645.034439:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772645/real 1790772645] req@ffff9c4ec7a7fb80 x1877759534498560/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772645.034456:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534498560/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772645.034471:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772645.034475:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772645.034481:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534498560:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772645.034490:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534498688:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772645.034518:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772645.034530:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772645/real 1790772645] req@ffff9c4ec7a7c700 x1877759534498688/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772645.034546:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534498688/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772645.034558:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772645.034563:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772645.034567:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534498688:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772645.096780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772645.096780:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772645.096795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772645.096796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772645.160792:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772645.160833:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772645.160888:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772645.288767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772645.288791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772645.352773:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772645.416807:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772645.480756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772645.480803:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772645.480803:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772645.544749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772645.608756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772645.608777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772645.608793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772645.736761:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772645.736796:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772645.736812:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772645.736822:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772645.800776:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772645.800790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772645.800804:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772645.800834:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772645.800843:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772645.800870:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772645.928782:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772645.992776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772646.056775:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772646.120755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772646.120772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772646.120821:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772646.120821:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772646.184748:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772646.184788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772646.184909:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772646.312737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772646.312803:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772646.376747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772646.440789:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772646.504753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772646.504767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772646.504797:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772646.568755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772646.632759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772646.632800:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772646.632800:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772646.760755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772646.760768:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772646.760804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772646.760804:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772646.824765:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772646.824800:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772646.824822:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772646.824836:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772646.824859:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772646.824870:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772646.952789:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772647.016752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772647.080744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772647.144780:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772647.144786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772647.144806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772647.144810:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772647.208811:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772647.208846:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772647.208869:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772647.336762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772647.336791:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772647.400770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772647.464987:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772647.528751:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772647.528782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772647.528835:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772647.592773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772647.656763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772647.656792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772647.656793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772647.784753:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772647.784765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772647.784785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772647.784798:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772647.848810:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772647.848835:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772647.848911:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772647.848955:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772647.848980:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772647.848999:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772647.976794:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772648.040765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772648.104741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772648.168780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772648.168783:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772648.168909:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772648.168949:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772648.232827:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772648.232830:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772648.233086:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772648.360776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772648.360785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772648.424770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772648.488763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772648.552753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772648.552789:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772648.552814:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772648.616777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772648.680778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772648.680778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772648.680785:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772648.808812:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772648.808840:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772648.808848:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772648.808868:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772648.872774:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772648.872806:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772648.872820:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772648.872842:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772648.872851:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772648.872856:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772649.000772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772649.064781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772649.128773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772649.192751:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772649.192785:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772649.192788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772649.192818:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772649.256757:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772649.256830:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772649.256885:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772649.384750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772649.384772:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772649.448735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772649.512785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772649.576772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772649.576776:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772649.576795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772649.640723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772649.704758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772649.704774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772649.704788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772649.832754:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772649.832762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772649.832833:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772649.832836:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772649.896798:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772649.896813:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772649.896820:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772649.896836:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772649.896877:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772649.896917:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772650.024773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772650.088763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772650.152759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772650.152804:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772650.152815:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772650.152822:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772650.152838:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1646 00000100:00080000:0.0:1790772650.152857:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772650.152904:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772650.152974:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534498816/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772650.153019:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772650.153025:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772650.153028:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772650.153035:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1646 00000100:00080000:0.0:1790772650.153041:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772650.153051:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772650.153067:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772650.153088:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534498816:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772650.153093:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534498944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772650.153125:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772650.153130:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772650.153134:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772650.153140:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1646 00000100:00080000:0.0:1790772650.153145:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772650.153151:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772650.153158:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772650.153174:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534499072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772650.153187:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772650.153193:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772650.153196:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772650.153203:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1646 00000100:00080000:0.0:1790772650.153209:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772650.153214:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772650.153222:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772650.153231:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772650.153242:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534499200/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772650.153257:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772650.153263:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772650.153266:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772650.153272:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1646 00000100:00080000:0.0:1790772650.153278:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772650.153285:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772650.153292:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772650.153315:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772650/real 1790772650] req@ffff9c4ec7a7d180 x1877759534498816/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772650.153316:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534499328/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772650.153366:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534498816/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772650.153372:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772650.153393:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772650.153401:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772650.153402:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772650.153413:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772650.153424:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534498816:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772650.153434:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1646 00000100:00080000:0.0:1790772650.153445:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772650.153457:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772650.153467:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772650.153473:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534498944:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772650.153493:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534499456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772650.153511:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772650.153516:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772650.153520:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772650.153527:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1646 00000100:00080000:0.0:1790772650.153534:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1790772650.153543:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772650.153548:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772650.153559:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772650.153577:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534499584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772650.153587:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772650/real 1790772650] req@ffff9c4ec7a7c700 x1877759534498944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772650.153643:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772650.153686:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772650.153691:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772650.153700:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1646 00000100:00100000:3.0:1790772650.153701:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534498944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772650.153708:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772650.153724:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772650.153735:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:3.0:1790772650.153747:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790772650.153755:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534499712/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772650.153762:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772650.153778:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534498944:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772650.153787:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772650.153797:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534499072:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772650.153799:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772650.153806:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772650.153816:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1646 00000100:00080000:0.0:1790772650.153822:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772650.153829:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772650.153838:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:3.0:1790772650.153849:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772650.153859:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534499840/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772650.153875:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772650/real 1790772650] req@ffff9c4ec7a7fb80 x1877759534499072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772650.153938:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534499072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772650.153952:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772650.153957:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772650.153962:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534499072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772650.153972:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534499200:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772650.154004:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772650.154019:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772650/real 1790772650] req@ffff9c4ec7a7e300 x1877759534499200/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772650.154035:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534499200/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772650.154048:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772650.154053:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772650.154057:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534499200:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772650.154065:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534499328:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772650.154086:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772650.154097:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772650/real 1790772650] req@ffff9c4ec7a7ca80 x1877759534499328/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772650.154113:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534499328/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772650.154145:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772650.154152:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772650.154160:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534499328:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772650.154196:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534499456:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772650.154240:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772650.154259:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772650/real 1790772650] req@ffff9c4ec7a7ed80 x1877759534499456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772650.154276:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534499456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772650.154299:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772650.154304:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772650.154309:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534499456:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772650.154318:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534499584:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772650.154341:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772650.154354:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772650/real 1790772650] req@ffff9c4ec7a7d880 x1877759534499584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772650.154370:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534499584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772650.154381:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772650.154384:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772650.154390:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534499584:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772650.154397:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534499712:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772650.154416:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772650.154428:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772650/real 1790772650] req@ffff9c4ec7a7c380 x1877759534499712/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772650.154442:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534499712/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772650.154455:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772650.154460:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772650.154466:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534499712:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772650.154478:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534499840:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772650.154515:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772650.154539:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772650/real 1790772650] req@ffff9c4ec7a7f480 x1877759534499840/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772650.154567:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534499840/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772650.154584:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772650.154589:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772650.154594:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534499840:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772650.216770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772650.216792:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772650.216825:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772650.216853:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772650.280827:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772650.280835:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772650.280978:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772650.408775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772650.408785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772650.472746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772650.536761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772650.600784:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772650.600798:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772650.600801:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772650.664773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772650.728761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772650.728792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772650.728832:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772650.856781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772650.856789:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772650.856802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772650.856845:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772650.920824:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772650.920835:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772650.920850:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772650.920850:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772650.920915:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772650.920944:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772651.048770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772651.112758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772651.176824:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772651.240743:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772651.240814:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772651.240827:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772651.240848:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772651.304895:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772651.304898:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772651.304986:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772651.432786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772651.432895:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772651.496786:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772651.560766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772651.624775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772651.624788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772651.624826:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772651.688795:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772651.752805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772651.752810:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772651.752822:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772651.880768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772651.880786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772651.880800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772651.880815:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772651.944818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772651.944830:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772651.944879:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772651.944888:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772651.944933:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772651.944934:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772652.072790:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772652.136791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772652.200782:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772652.264797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772652.264864:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772652.264882:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772652.264905:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772652.328864:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772652.328870:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772652.328922:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772652.456784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772652.456801:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772652.520772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772652.584790:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772652.648773:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772652.648809:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772652.648852:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772652.712773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772652.776762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772652.776797:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772652.776860:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772652.904771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772652.904799:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772652.904816:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772652.904821:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772652.968787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772652.968818:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772652.968845:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772652.968849:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772652.968895:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772652.968917:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772653.096746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772653.160730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772653.224747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772653.288790:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772653.288790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772653.288893:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772653.288894:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772653.352873:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772653.352887:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772653.352911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772653.480755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772653.480799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772653.544755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772653.608803:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772653.672776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772653.672786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772653.672794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772653.736755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772653.800758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772653.800797:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772653.800803:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772653.928798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772653.928882:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772653.928914:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772653.928915:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772653.992827:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772653.992829:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772653.992871:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772653.992895:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772653.992924:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772653.992935:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772654.120766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772654.184775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790772654.248791:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772654.312775:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772654.312826:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772654.312838:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772654.312839:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772654.376823:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772654.376846:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772654.376921:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772654.504779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772654.504783:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772654.568785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772654.632919:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772654.696838:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772654.696847:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772654.696847:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772654.760773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772654.824756:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772654.824840:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772654.824840:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772654.952763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772654.952852:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772654.952853:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772654.952853:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772655.016824:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772655.016833:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772655.016839:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772655.016839:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772655.016896:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772655.016903:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772655.144751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772655.208776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772655.272827:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772655.272845:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772655.272853:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772655.272876:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1651 00000100:00080000:3.0:1790772655.272899:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772655.272904:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772655.272927:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772655.273004:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534499968/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772655.273064:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772655.273072:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772655.273075:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772655.273082:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1651 00000100:00080000:3.0:1790772655.273090:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772655.273099:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772655.273116:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772655.273140:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534499968:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772655.273150:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534500096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772655.273174:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772655.273180:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772655.273183:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772655.273194:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1651 00000100:00080000:3.0:1790772655.273198:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772655.273205:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772655.273212:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772655.273232:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534500224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772655.273247:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772655.273253:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772655.273257:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772655.273263:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1651 00000100:00080000:3.0:1790772655.273268:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772655.273275:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772655.273283:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772655.273299:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534500352/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772655.273313:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772655.273319:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772655.273325:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772655.273326:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534500096:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772655.273350:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1651 00000100:00080000:3.0:1790772655.273356:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772655.273362:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772655.273373:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772655.273409:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534500480/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772655.273412:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534500224:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772655.273455:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772655.273466:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772655.273471:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772655.273482:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1651 00000100:00100000:2.0:1790772655.273490:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534500352:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772655.273492:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772655.273514:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772655.273526:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772655.273548:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534500480:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772655.273552:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534500608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772655.273581:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772655.273587:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772655.273593:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772655.273606:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1651 00000100:00100000:2.0:1790772655.273615:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534500608:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772655.273624:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772655.273643:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772655.273720:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772655.273755:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534500736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772655.273820:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772655.273828:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772655.273833:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772655.273842:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1651 00000100:00080000:3.0:1790772655.273853:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772655.273866:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772655.273877:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772655.273901:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534500736:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772655.273907:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534500864/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772655.273942:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772655.273950:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772655.273954:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772655.273964:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1651 00000100:00080000:3.0:1790772655.273974:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772655.273989:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772655.274000:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772655.274010:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534500864:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772655.274034:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534500992/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772655.274104:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534500992:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772655.336806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772655.336809:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772655.336814:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772655.336875:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772655.400862:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772655.400914:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772655.400965:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772655.528756:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772655.528822:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772655.592812:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772655.656837:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772655.720830:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772655.720851:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772655.720855:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772655.784755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772655.848756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772655.848820:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772655.848834:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772655.976809:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772655.976828:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772655.976829:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772655.976832:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772656.040812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772656.040814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772656.040857:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772656.040868:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772656.040875:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772656.040925:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772656.168809:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772656.232793:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772656.296801:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772656.360813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772656.360833:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772656.360835:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772656.360860:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772656.424792:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772656.424853:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772656.424978:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772656.552795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772656.552896:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772656.616741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772656.680816:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772656.744781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772656.744794:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772656.744818:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772656.808791:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772656.872786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772656.872826:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772656.872833:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772657.000791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772657.000847:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772657.000866:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772657.000871:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772657.064830:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772657.064846:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772657.064862:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772657.064875:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772657.064928:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772657.064942:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772657.192812:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772657.256773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772657.320795:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772657.384808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772657.384849:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772657.384881:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772657.384885:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772657.448880:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772657.448886:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772657.448934:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772657.576787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772657.576847:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772657.640797:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772657.704811:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772657.768822:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772657.768846:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772657.768846:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772657.832785:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772657.896831:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772657.896831:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772657.896845:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772658.024798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772658.024820:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772658.024834:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772658.024835:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772658.088768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772658.088800:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772658.088817:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772658.088823:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772658.088843:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772658.088868:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772658.216780:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772658.280783:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772658.344790:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:1.0:1790772658.345138:0:2386:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:1.0:1790772658.345240:0:2386:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1790772658.345275:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772658.345380:0:2386:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:1.0:1790772658.345414:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772658.345440:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790772658.345452:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772655/real 1790772658] req@ffff9c4ef73c0000 x1877759534500992/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790772658.345469:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772658.345500:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772658.345518:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790772658.345542:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534500992/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790772658.345547:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772658.345577:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:2.0:1790772658.345587:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772658.345616:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772658.345626:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534500992:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772658.345644:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772655/real 1790772658] req@ffff9c4ef73c1c00 x1877759534500864/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772658.345757:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534500864/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772658.345783:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772658.345791:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772658.345801:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534500864:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772658.345818:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772655/real 1790772658] req@ffff9c4ef73c2a00 x1877759534500736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772658.345859:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534500736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772658.345878:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772658.345887:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772658.345895:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534500736:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772658.345910:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772655/real 1790772658] req@ffff9c4ef73c0700 x1877759534500608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772658.345937:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534500608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772658.345970:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772658.345980:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772658.345988:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534500608:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772658.346003:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772655/real 1790772658] req@ffff9c4ef73c2300 x1877759534500480/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772658.346031:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534500480/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772658.346050:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772658.346058:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772658.346066:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534500480:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772658.346079:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772655/real 1790772658] req@ffff9c4ef73c3800 x1877759534500096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772658.346106:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534500096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772658.346134:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772658.346143:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772658.346150:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534500096:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772658.346164:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772655/real 1790772658] req@ffff9c4ef73c0a80 x1877759534500224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772658.346205:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534500224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772658.346246:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772658.346256:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772658.346264:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534500224:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772658.346277:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772655/real 1790772658] req@ffff9c4ef73c1500 x1877759534500352/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772658.346305:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534500352/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772658.346326:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772658.346334:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772658.346351:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534500352:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772658.346370:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772655/real 1790772658] req@ffff9c4ef73c2d80 x1877759534499968/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772658.346398:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534499968/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772658.346416:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772658.346425:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772658.346434:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534499968:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772658.408802:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772658.408817:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772658.408834:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772658.408838:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772658.472785:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772658.472860:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772658.472919:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772658.600807:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772658.600807:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772658.664802:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772658.728812:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772658.792780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772658.792799:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772658.792817:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772658.856775:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772658.920802:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772658.920830:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772658.920831:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772659.048776:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772659.048811:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772659.048819:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772659.048844:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772659.112785:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772659.112818:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772659.112823:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772659.112838:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772659.112855:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772659.112887:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772659.240789:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772659.304774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772659.368793:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772659.432791:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772659.432826:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772659.432826:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772659.432833:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772659.496814:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772659.496851:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772659.496922:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772659.624789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772659.624806:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772659.688820:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772659.752834:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772659.816784:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772659.816796:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772659.816822:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772659.880811:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772659.944787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772659.944799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772659.944811:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772660.072779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772660.072813:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772660.072829:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772660.072832:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772660.136744:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772660.136827:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772660.136829:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772660.136830:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772660.136830:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772660.136886:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772660.264781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772660.328792:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772660.392804:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772660.392814:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772660.392826:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772660.392833:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772660.392851:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1656 00000100:00080000:3.0:1790772660.392869:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772660.392889:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772660.392956:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534501120/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772660.392999:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772660.393005:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772660.393008:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772660.393013:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1656 00000100:00080000:3.0:1790772660.393018:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772660.393025:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772660.393041:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772660.393062:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534501248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772660.393069:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534501120:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772660.393092:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772660.393098:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772660.393101:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772660.393108:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1656 00000100:00080000:3.0:1790772660.393116:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772660.393127:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772660.393137:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772660.393165:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534501376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772660.393187:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772660.393196:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772660.393201:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772660.393209:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772660.393220:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1656 00000100:00080000:3.0:1790772660.393231:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772660.393243:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772660.393255:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772660.393269:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772660/real 1790772660] req@ffff9c4ef73c0e00 x1877759534501120/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772660.393285:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534501504/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772660.393312:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772660.393320:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772660.393324:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772660.393332:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534501120/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772660.393340:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1656 00000100:00080000:3.0:1790772660.393354:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772660.393372:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772660.393375:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790772660.393386:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:2.0:1790772660.393391:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772660.393406:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534501120:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772660.393418:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534501632/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772660.393467:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534501248:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772660.393497:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772660.393506:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772660.393518:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790772660.393530:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772660.393535:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1656 00000100:00080000:3.0:1790772660.393548:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772660.393564:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772660.393574:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772660/real 1790772660] req@ffff9c4ef73c2d80 x1877759534501248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772660.393580:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772660.393617:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534501760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772660.393624:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534501248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772660.393722:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772660.393731:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772660.393735:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772660.393746:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1656 00000100:00080000:3.0:1790772660.393755:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772660.393767:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772660.393778:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772660.393806:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790772660.393810:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534501888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772660.393824:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772660.393843:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534501248:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772660.393855:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772660.393868:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1790772660.393869:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534501376:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772660.393876:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772660.393891:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1656 00000100:00080000:3.0:1790772660.393902:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772660.394322:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772660.394336:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772660.394364:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534502016/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772660.394386:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772660.394393:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772660.394397:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772660.394406:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1656 00000100:00080000:3.0:1790772660.394414:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772660.394429:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772660.394429:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772660.394448:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772660.394476:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534502144/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772660.394481:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772660/real 1790772660] req@ffff9c4ef73c1500 x1877759534501376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772660.394544:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534501376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772660.394569:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772660.394578:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772660.394587:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534501376:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772660.394602:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534501504:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772660.394723:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772660.394755:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772660/real 1790772660] req@ffff9c4ef73c0a80 x1877759534501504/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772660.394812:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534501504/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772660.394836:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772660.394844:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772660.394852:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534501504:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772660.394901:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534501632:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772660.394958:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772660.394981:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772660/real 1790772660] req@ffff9c4ef73c3800 x1877759534501632/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772660.395004:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534501632/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772660.395023:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772660.395030:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772660.395038:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534501632:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772660.395051:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534501760:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772660.395110:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772660.395137:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772660/real 1790772660] req@ffff9c4ef73c2300 x1877759534501760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772660.395163:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534501760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772660.395195:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772660.395203:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772660.395211:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534501760:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772660.395224:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534501888:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772660.395278:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772660.395307:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772660/real 1790772660] req@ffff9c4ef73c0700 x1877759534501888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772660.395333:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534501888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772660.395352:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772660.395359:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772660.395368:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534501888:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772660.395381:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534502016:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772660.395417:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772660.395434:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772660/real 1790772660] req@ffff9c4ef73c2a00 x1877759534502016/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772660.395450:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534502016/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772660.395465:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772660.395470:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772660.395474:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534502016:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772660.395483:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534502144:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772660.395503:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772660.395515:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772660/real 1790772660] req@ffff9c4ef73c1c00 x1877759534502144/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772660.395536:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534502144/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772660.395557:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772660.395564:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772660.395572:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534502144:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772660.456842:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772660.456857:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772660.456863:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772660.456865:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772660.520866:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772660.520935:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772660.520960:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772660.648816:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772660.648822:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772660.712788:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772660.776804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772660.840810:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772660.840825:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772660.840841:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772660.904790:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772660.968776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772660.968795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772660.968801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772661.096787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772661.096801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772661.096809:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772661.096825:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772661.160773:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772661.160811:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772661.160819:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772661.160838:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772661.160871:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772661.160891:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772661.288749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772661.352779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772661.416795:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772661.480767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772661.480781:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772661.480821:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772661.480822:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772661.544823:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772661.544853:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772661.544906:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772661.672813:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772661.672819:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772661.736794:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772661.800894:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772661.864767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772661.864776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772661.864822:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772661.928784:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772661.992796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772661.992796:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772661.992814:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772662.120799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772662.120824:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772662.120826:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772662.120831:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772662.184766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772662.184807:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772662.184822:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772662.184829:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772662.184861:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772662.184862:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772662.312781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772662.376755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772662.440740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772662.504782:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772662.504797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772662.504819:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772662.504826:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772662.568789:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772662.568891:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772662.568943:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772662.696779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772662.696792:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772662.760781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772662.824779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772662.888803:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772662.888810:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772662.888846:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772662.952747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772663.016804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772663.016818:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772663.016831:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772663.144780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772663.144789:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772663.144821:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772663.144836:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772663.208812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772663.208829:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772663.208865:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772663.208887:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772663.208918:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772663.208953:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772663.336744:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772663.400793:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772663.464807:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772663.528834:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772663.528835:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772663.528850:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772663.528851:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772663.592847:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772663.592864:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772663.592991:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772663.720768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772663.720803:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772663.784748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772663.848811:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772663.912771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772663.912824:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772663.912825:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772663.976779:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772664.040767:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772664.040795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772664.040808:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772664.168776:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772664.168798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772664.168809:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772664.168821:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772664.232785:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772664.232808:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772664.232811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772664.232862:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772664.232901:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772664.232918:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772664.360757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772664.424753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772664.488780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772664.552788:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772664.552799:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772664.552800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772664.552806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772664.616791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772664.616906:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772664.616920:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772664.744775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772664.744784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772664.808762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772664.872763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772664.936753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772664.936802:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772664.936808:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772665.000753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772665.064771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772665.064778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772665.064818:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772665.192796:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772665.192829:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772665.192842:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772665.192854:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772665.256827:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772665.256880:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772665.256881:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772665.256887:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772665.256962:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772665.256990:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772665.384766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772665.448761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772665.512797:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772665.512801:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772665.512822:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772665.512830:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772665.512844:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1661 00000100:00080000:3.0:1790772665.512860:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772665.512875:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772665.512925:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534502272/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772665.512966:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772665.512972:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772665.512975:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772665.512981:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1661 00000100:00080000:3.0:1790772665.512986:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772665.512992:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772665.513005:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772665.513027:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534502400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772665.513040:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534502272:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772665.513055:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772665.513063:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772665.513066:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772665.513072:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1661 00000100:00080000:3.0:1790772665.513079:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772665.513087:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772665.513096:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772665.513114:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534502528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772665.513129:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772665.513134:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772665.513138:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772665.513143:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1661 00000100:00080000:3.0:1790772665.513150:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772665.513158:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772665.513166:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:2.0:1790772665.513177:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772665.513185:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534502656/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772665.513200:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772665.513206:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772665.513209:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772665.513219:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1661 00000100:00080000:3.0:1790772665.513227:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772665.513233:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772665.513240:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772665.513258:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534502784/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772665.513275:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772665/real 1790772665] req@ffff9c4ef73c0000 x1877759534502272/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772665.513276:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772665.513303:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772665.513315:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772665.513340:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1661 00000100:00080000:3.0:1790772665.513349:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772665.513355:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534502272/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772665.513367:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772665.513393:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772665.513402:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772665.513418:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772665.513419:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534502912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772665.513439:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534502272:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772665.513451:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772665.513459:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772665.513463:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772665.513471:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1661 00000100:00080000:3.0:1790772665.513476:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772665.513481:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772665.513492:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772665.513497:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534502400:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772665.513513:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534503040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772665.513532:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772665.513537:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1790772665.513549:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772665.513550:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772665.513569:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1661 00000100:00080000:3.0:1790772665.513576:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772665.513613:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772665/real 1790772665] req@ffff9c4ef73c1c00 x1877759534502400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772665.513636:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772665.513647:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534502400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772665.513716:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:2.0:1790772665.513718:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772665.513731:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772665.513740:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534502400:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772665.513753:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534502528:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772665.513766:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534503168/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772665.513794:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772665.513815:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772665/real 1790772665] req@ffff9c4ef73c2a00 x1877759534502528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772665.513821:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772665.513851:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772665.513864:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772665.513873:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534502528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772665.513877:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1661 00000100:00080000:3.0:1790772665.513895:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772665.513909:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772665.513919:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772665.513928:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:3.0:1790772665.513936:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772665.513937:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534502528:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772665.513964:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534502656:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772665.513974:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534503296/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772665.514002:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772665.514025:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772665/real 1790772665] req@ffff9c4ef73c0700 x1877759534502656/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772665.514052:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534502656/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772665.514066:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772665.514071:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772665.514076:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534502656:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772665.514083:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534502784:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772665.514105:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772665.514117:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772665/real 1790772665] req@ffff9c4ef73c2300 x1877759534502784/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772665.514132:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534502784/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772665.514145:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772665.514149:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772665.514153:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534502784:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772665.514161:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534502912:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772665.514181:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772665.514193:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772665/real 1790772665] req@ffff9c4ef73c3800 x1877759534502912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772665.514208:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534502912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772665.514238:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772665.514243:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772665.514247:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534502912:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772665.514281:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534503040:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772665.514304:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772665.514317:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772665/real 1790772665] req@ffff9c4ef73c0a80 x1877759534503040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772665.514333:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534503040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772665.514344:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772665.514348:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772665.514352:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534503040:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772665.514360:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534503168:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772665.514400:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772665.514414:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772665/real 1790772665] req@ffff9c4ef73c1500 x1877759534503168/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772665.514430:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534503168/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772665.514452:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772665.514457:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772665.514463:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534503168:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772665.514472:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534503296:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772665.514494:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772665.514507:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772665/real 1790772665] req@ffff9c4ef73c2d80 x1877759534503296/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772665.514522:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534503296/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772665.514535:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772665.514540:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772665.514544:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534503296:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772665.576813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772665.576816:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772665.576825:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772665.576829:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772665.640823:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772665.640863:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772665.640930:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772665.768774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772665.768807:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772665.832790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772665.896755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772665.960769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772665.960772:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772665.960818:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772666.024768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772666.088772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772666.088793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772666.088802:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772666.216775:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772666.216795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772666.216797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772666.216800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772666.280757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772666.280809:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772666.280839:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772666.280843:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772666.280865:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772666.280877:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772666.408769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772666.472753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772666.536767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772666.600772:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772666.600784:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772666.600797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772666.600811:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772666.664796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772666.664849:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772666.664911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772666.792762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772666.792762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772666.856750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772666.920754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772666.984791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772666.984795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772666.984795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772667.048717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772667.112749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772667.112784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772667.112785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772667.240753:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772667.240790:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772667.240791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772667.240793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772667.304794:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772667.304795:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772667.304799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772667.304811:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772667.304854:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772667.304868:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772667.432752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772667.496756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772667.560778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772667.624763:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772667.624765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772667.624796:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772667.624810:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772667.688789:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772667.688851:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772667.688912:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772667.816745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772667.816772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772667.880762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772667.944805:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772668.008765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772668.008772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772668.008785:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772668.072792:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772668.136762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772668.136772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772668.136780:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772668.264769:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772668.264797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772668.264802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772668.264803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772668.328782:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772668.328796:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772668.328806:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772668.328819:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772668.328849:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772668.328858:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772668.456758:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772668.520754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772668.584768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772668.648761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772668.648765:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772668.648772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772668.648805:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772668.712823:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772668.712834:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772668.712841:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772668.840771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772668.840771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772668.904762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772668.968876:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772669.032781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772669.032797:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772669.032798:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772669.096762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772669.160749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772669.160776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772669.160791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772669.288769:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772669.288795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772669.288795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772669.288801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772669.352780:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772669.352811:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772669.352811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772669.352839:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772669.352855:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772669.352886:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772669.480753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772669.544773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772669.608768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772669.672767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772669.672773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772669.672817:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772669.672817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772669.736764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772669.736791:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772669.736832:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772669.864757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772669.864779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772669.928749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772669.992772:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772670.056757:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772670.056777:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772670.056790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772670.120829:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772670.184735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772670.184784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772670.184785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772670.312728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772670.312735:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772670.312741:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772670.312743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772670.376728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772670.376730:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772670.376749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772670.376761:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772670.376785:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772670.376790:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772670.504719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772670.568723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772670.632776:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772670.632836:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772670.632844:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772670.632848:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772670.632857:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1666 00000100:00080000:1.0:1790772670.632866:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772670.632877:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772670.632911:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534503424/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772670.632949:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772670.632953:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772670.632954:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772670.632958:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1666 00000100:00080000:1.0:1790772670.632960:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772670.632963:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772670.632971:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772670.632981:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534503552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772670.632988:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772670.632990:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772670.632992:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772670.632994:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1666 00000100:00080000:1.0:1790772670.632996:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772670.632999:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772670.633003:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772670.633010:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534503680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772670.633027:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772670.633030:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772670.633031:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772670.633035:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1666 00000100:00080000:1.0:1790772670.633038:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772670.633042:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534503424:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772670.633043:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772670.633051:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772670.633068:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534503808/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772670.633077:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772670.633079:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772670.633081:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772670.633084:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1666 00000100:00080000:1.0:1790772670.633088:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772670.633091:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772670.633094:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772670.633103:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534503936/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772670.633109:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772670.633116:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772670.633119:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772670.633121:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772670.633125:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1666 00000100:00080000:1.0:1790772670.633128:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772670.633133:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772670.633142:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772670.633161:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534504064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772670.633164:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772670/real 1790772670] req@ffff9c4efac4e680 x1877759534503424/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772670.633192:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772670.633195:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772670.633197:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772670.633198:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534503424/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772670.633202:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1666 00000100:00080000:1.0:1790772670.633206:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772670.633213:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772670.633217:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790772670.633218:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772670.633225:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790772670.633229:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534504192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772670.633230:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534503424:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772670.633244:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534503552:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772670.633245:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772670.633252:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772670.633254:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772670.633258:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1666 00000100:00080000:1.0:1790772670.633261:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772670.633267:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772670.633271:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772670.633274:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772670.633284:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772670/real 1790772670] req@ffff9c4efac4ea00 x1877759534503552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772670.633286:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534504320/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772670.633314:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772670.633318:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1790772670.633318:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534503552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772670.633320:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772670.633327:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1666 00000100:00080000:1.0:1790772670.633333:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772670.633339:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772670.633343:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:2.0:1790772670.633344:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772670.633349:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790772670.633353:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534504448/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772670.633356:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534503552:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772670.633368:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534503680:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772670.633385:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772670.633393:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772670/real 1790772670] req@ffff9c4efac4c700 x1877759534503680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772670.633414:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534503680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772670.633420:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772670.633422:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772670.633425:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534503680:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772670.633445:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534503808:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772670.633457:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772670.633463:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772670/real 1790772670] req@ffff9c4efac4ed80 x1877759534503808/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772670.633472:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534503808/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772670.633497:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772670.633500:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772670.633502:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534503808:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772670.633507:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534503936:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772670.633533:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772670.633540:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772670/real 1790772670] req@ffff9c4efac4dc00 x1877759534503936/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772670.633548:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534503936/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772670.633559:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772670.633562:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772670.633564:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534503936:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772670.633567:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534504064:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772670.633577:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772670.633584:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772670/real 1790772670] req@ffff9c4efac4f800 x1877759534504064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772670.633591:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534504064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772670.633613:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772670.633615:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772670.633617:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534504064:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772670.633622:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534504192:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772670.633632:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772670.633638:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772670/real 1790772670] req@ffff9c4efac4f480 x1877759534504192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772670.633646:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534504192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772670.633678:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772670.633682:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772670.633687:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534504192:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772670.633693:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534504320:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772670.633716:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772670.633727:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772670/real 1790772670] req@ffff9c4efac4e300 x1877759534504320/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772670.633736:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534504320/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772670.633744:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772670.633746:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772670.633749:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534504320:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772670.633753:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534504448:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772670.633775:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772670.633782:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772670/real 1790772670] req@ffff9c4efac4df80 x1877759534504448/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772670.633789:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534504448/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772670.633796:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772670.633798:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772670.633800:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534504448:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772670.696729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772670.696732:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772670.696806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772670.696845:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772670.760802:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772670.760810:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772670.760861:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772670.888714:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772670.888773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772670.952822:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772671.016777:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772671.080816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772671.080837:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772671.080837:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772671.144749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772671.208783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772671.208789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772671.208817:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772671.336786:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772671.336788:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772671.336805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772671.336811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772671.400785:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772671.400843:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772671.400855:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772671.400866:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772671.400870:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772671.400942:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772671.528765:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772671.592759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772671.656780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772671.720848:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772671.720848:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772671.720850:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772671.720859:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772671.784812:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772671.784851:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772671.784863:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772671.912873:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772671.912875:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772671.976767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772672.040795:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772672.104762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772672.104766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772672.104795:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772672.168736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772672.232767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772672.232772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772672.232780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772672.360770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772672.360775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772672.360799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772672.360805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772672.424764:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772672.424778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772672.424809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772672.424813:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772672.424831:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772672.424850:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772672.552756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772672.616747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772672.680987:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772672.744765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772672.744780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772672.744788:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772672.744813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772672.808807:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772672.808856:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772672.808891:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772672.936752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772672.936783:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772673.000769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772673.064780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772673.128779:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772673.128786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772673.128790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772673.192755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772673.256754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772673.256777:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772673.256789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772673.384753:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772673.384773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772673.384804:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772673.384816:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772673.448845:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772673.448855:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772673.448866:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772673.448866:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772673.448909:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772673.448953:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772673.576800:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772673.640771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772673.704803:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772673.768787:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772673.768804:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772673.768815:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772673.768823:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772673.832783:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772673.832852:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772673.832985:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772673.960772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772673.960792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772674.024762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772674.088788:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772674.152788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772674.152848:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772674.152852:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772674.216739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772674.280786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772674.280786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772674.280796:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772674.408795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772674.408807:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772674.408856:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772674.408858:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772674.472807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772674.472815:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772674.472858:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772674.472859:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772674.472923:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772674.472930:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772674.600756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772674.664756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772674.728799:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772674.792768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772674.792783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772674.792817:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772674.792823:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772674.856859:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772674.856881:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772674.856982:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772674.984762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772674.984772:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772675.048758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772675.112779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772675.176743:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772675.176787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772675.176817:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772675.240783:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772675.304761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772675.304772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772675.304830:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772675.432804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772675.432805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772675.432806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772675.432813:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772675.496783:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772675.496797:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772675.496807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772675.496807:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772675.496839:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772675.496859:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772675.624749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772675.688766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790772675.752818:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772675.752820:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772675.752836:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772675.752844:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772675.752864:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1671 00000100:00080000:1.0:1790772675.752886:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772675.752912:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772675.753015:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534504576/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772675.753080:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772675.753087:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772675.753090:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772675.753097:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1671 00000100:00080000:1.0:1790772675.753104:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772675.753114:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772675.753132:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772675.753160:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534504704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772675.753172:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534504576:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772675.753200:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772675.753206:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772675.753209:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772675.753216:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1671 00000100:00080000:1.0:1790772675.753221:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772675.753227:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772675.753235:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772675.753252:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534504832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772675.753265:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772675.753272:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772675.753277:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772675.753286:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1671 00000100:00080000:1.0:1790772675.753295:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772675.753297:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772675.753315:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772675.753330:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772675.753360:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534504960/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772675.753376:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772675/real 1790772675] req@ffff9c4efac4fb80 x1877759534504576/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772675.753414:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772675.753430:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772675.753436:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772675.753446:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534504576/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772675.753450:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1671 00000100:00080000:1.0:1790772675.753468:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772675.753475:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772675.753485:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772675.753488:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772675.753502:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534504576:192.168.204.150@tcp:250:kworker.0 00000008:00100000:1.0:1790772675.753504:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772675.753552:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534505088/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772675.753564:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534504704:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772675.753575:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772675.753585:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772675.753591:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772675.753602:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1671 00000100:00080000:1.0:1790772675.753614:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772675.753629:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772675.753635:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772675.753698:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772675.753724:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772675/real 1790772675] req@ffff9c4efac4df80 x1877759534504704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772675.753729:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534505216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772675.753787:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772675.753797:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1790772675.753798:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534504704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772675.753804:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772675.753814:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1671 00000100:00080000:1.0:1790772675.753820:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772675.753832:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772675.753850:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772675.753871:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790772675.753882:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534505344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772675.753884:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772675.753923:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534504704:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772675.753931:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772675.753943:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1790772675.753950:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534504832:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772675.753951:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772675.753972:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1671 00000100:00080000:1.0:1790772675.753983:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772675.753998:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772675.754010:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:2.0:1790772675.754022:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772675.754043:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534505472/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772675.754060:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772675/real 1790772675] req@ffff9c4efac4e300 x1877759534504832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772675.754064:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772675.754078:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772675.754084:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772675.754101:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1671 00000100:00080000:1.0:1790772675.754112:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772675.754125:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772675.754125:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534504832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772675.754139:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:2.0:1790772675.754155:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790772675.754166:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534505600/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772675.754170:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772675.754186:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534504832:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772675.754209:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534504960:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772675.754247:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772675.754264:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772675/real 1790772675] req@ffff9c4efac4f480 x1877759534504960/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772675.754280:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534504960/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772675.754295:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772675.754300:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772675.754306:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534504960:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772675.754334:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534505088:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772675.754357:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772675.754371:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772675/real 1790772675] req@ffff9c4efac4f800 x1877759534505088/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772675.754419:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534505088/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772675.754433:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772675.754438:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772675.754443:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534505088:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772675.754451:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534505216:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772675.754481:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772675.754495:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772675/real 1790772675] req@ffff9c4efac4dc00 x1877759534505216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772675.754511:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534505216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772675.754538:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772675.754544:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772675.754551:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534505216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772675.754565:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534505344:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772675.754606:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772675.754629:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772675/real 1790772675] req@ffff9c4efac4ed80 x1877759534505344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772675.754646:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534505344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772675.754698:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772675.754718:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772675.754729:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534505344:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772675.754738:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534505472:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772675.754768:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772675.754784:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772675/real 1790772675] req@ffff9c4efac4c700 x1877759534505472/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772675.754800:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534505472/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772675.754815:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772675.754819:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772675.754824:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534505472:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772675.754831:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534505600:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772675.754851:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772675.754863:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772675/real 1790772675] req@ffff9c4efac4ea00 x1877759534505600/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772675.754877:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534505600/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772675.754893:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772675.754900:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772675.754907:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534505600:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772675.816788:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772675.816789:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772675.816801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772675.816802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772675.880787:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772675.880829:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772675.880863:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772676.008771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772676.008771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772676.072817:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772676.136872:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772676.200745:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772676.200755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772676.200759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772676.264731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772676.328781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772676.328803:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772676.328804:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772676.456768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772676.456793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772676.456794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772676.456803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772676.520774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772676.520777:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772676.520784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772676.520799:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772676.520840:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772676.520864:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772676.648749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772676.712772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772676.776768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772676.840777:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772676.840777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772676.840790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772676.840790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772676.904759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772676.904769:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772676.904887:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772677.032791:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772677.032791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772677.096761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772677.160774:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772677.224763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772677.224768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772677.224774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772677.288749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772677.352762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772677.352792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772677.352830:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772677.480751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772677.480774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772677.480790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772677.480791:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772677.544748:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772677.544789:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772677.544820:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772677.544820:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772677.544855:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772677.544857:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772677.672782:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772677.736757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772677.800799:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772677.864819:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772677.864862:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772677.864910:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772677.864914:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772677.928802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772677.928903:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772677.928995:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772678.056750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772678.056772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772678.120758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772678.184815:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772678.248768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772678.248802:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772678.248805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772678.312741:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772678.376768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772678.376809:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772678.376822:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772678.504779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772678.504793:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772678.504819:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772678.504819:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772678.568762:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772678.568795:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772678.568815:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772678.568833:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772678.568884:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772678.568892:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772678.696806:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772678.760751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772678.824750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772678.888787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772678.888787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772678.888787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772678.888832:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772678.952772:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772678.952875:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772678.952894:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772679.080760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772679.080772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772679.144751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772679.208755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772679.272794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772679.272794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772679.273760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772679.336735:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772679.400776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772679.400780:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772679.400790:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772679.528841:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772679.528848:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772679.528853:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772679.528855:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772679.592802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772679.592804:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772679.592807:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772679.592823:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772679.592929:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772679.592933:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772679.720902:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772679.784746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772679.848757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772679.912834:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772679.912885:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772679.912889:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772679.912890:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772679.976847:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772679.976860:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772679.976967:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772680.104805:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772680.104814:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772680.168756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772680.232814:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772680.296767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772680.296770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772680.296839:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772680.360782:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772680.424797:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772680.424797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772680.424797:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772680.552751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772680.552796:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772680.552805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772680.552838:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772680.616826:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772680.616838:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772680.616839:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772680.616840:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772680.616915:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772680.616921:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772680.744802:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772680.808760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772680.872759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772680.872814:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772680.872828:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772680.872839:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772680.872857:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1676 00000100:00080000:1.0:1790772680.872874:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772680.872896:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772680.872979:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534505728/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772680.873019:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772680.873025:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772680.873028:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772680.873033:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1676 00000100:00080000:1.0:1790772680.873039:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772680.873046:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772680.873062:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772680.873084:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534505856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772680.873091:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534505728:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772680.873118:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772680.873123:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772680.873126:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772680.873134:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1676 00000100:00080000:1.0:1790772680.873139:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772680.873146:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772680.873165:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772680.873184:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534505984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772680.873198:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772680.873202:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772680.873207:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772680.873215:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1676 00000100:00080000:1.0:1790772680.873221:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772680.873228:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772680.873237:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772680.873238:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772680.873260:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534506112/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772680.873276:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772680.873308:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772680.873311:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772680.873320:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1676 00000100:00080000:1.0:1790772680.873328:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772680.873336:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772680.873346:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772680/real 1790772680] req@ffff9c4efac4e680 x1877759534505728/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772680.873349:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772680.873381:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534506240/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772680.873392:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534505728/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772680.873408:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772680.873424:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772680.873431:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772680.873447:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1676 00000100:00080000:2.0:1790772680.873451:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772680.873454:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772680.873470:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772680.873471:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772680.873485:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772680.873488:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534505728:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772680.873510:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534506368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772680.873524:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772680.873530:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772680.873534:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772680.873543:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534505856:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772680.873544:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1676 00000100:00080000:1.0:1790772680.873557:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772680.873564:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772680.873572:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772680.873594:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534506496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772680.873607:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772680.873625:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772680.873631:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772680.873634:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772680.873642:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772680/real 1790772680] req@ffff9c4efac4ea00 x1877759534505856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772680.873647:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1676 00000100:00080000:1.0:1790772680.873706:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772680.873713:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534505856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772680.873722:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772680.873743:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:2.0:1790772680.873765:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772680.873776:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790772680.873782:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534506624/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772680.873788:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534505856:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772680.873824:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772680.873831:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534505984:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772680.873834:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772680.873843:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772680.873855:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1676 00000100:00080000:1.0:1790772680.873862:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772680.873872:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772680.873881:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772680.873903:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534506752/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772680.873913:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772680.873949:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772680/real 1790772680] req@ffff9c4efac4c700 x1877759534505984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772680.873968:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534505984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772680.873982:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772680.873987:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772680.873992:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534505984:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772680.874003:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534506112:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772680.874032:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772680.874045:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772680/real 1790772680] req@ffff9c4efac4ed80 x1877759534506112/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772680.874061:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534506112/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772680.874075:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772680.874079:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772680.874083:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534506112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772680.874090:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534506240:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772680.874111:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772680.874122:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772680/real 1790772680] req@ffff9c4efac4dc00 x1877759534506240/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772680.874137:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534506240/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772680.874149:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772680.874153:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772680.874158:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534506240:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772680.874165:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534506368:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772680.874186:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772680.874199:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772680/real 1790772680] req@ffff9c4efac4f800 x1877759534506368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772680.874215:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534506368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772680.874235:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772680.874239:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772680.874243:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534506368:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772680.874272:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534506496:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772680.874392:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772680.874409:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772680/real 1790772680] req@ffff9c4efac4f480 x1877759534506496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772680.874426:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534506496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772680.874438:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772680.874443:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772680.874448:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534506496:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772680.874456:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534506624:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772680.874477:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772680.874489:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772680/real 1790772680] req@ffff9c4efac4e300 x1877759534506624/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772680.874504:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534506624/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772680.874527:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772680.874532:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772680.874537:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534506624:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772680.874544:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534506752:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772680.874563:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772680.874574:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772680/real 1790772680] req@ffff9c4efac4df80 x1877759534506752/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772680.874589:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534506752/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772680.874602:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772680.874606:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772680.874610:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534506752:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772680.936791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772680.936791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772680.936816:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772680.936816:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772681.000805:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772681.000814:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772681.000954:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772681.128756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772681.128767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772681.192780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772681.256801:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772681.320757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772681.320781:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772681.320782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772681.384750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772681.448762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772681.448782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772681.448785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772681.576851:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772681.576894:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772681.577018:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772681.577124:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772681.640784:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772681.640786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772681.640794:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772681.640830:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772681.640851:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772681.640876:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772681.768754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772681.832741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772681.896747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772681.960770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772681.960801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772681.960805:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772681.960805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772682.024772:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772682.024789:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772682.024922:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772682.152798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772682.152798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772682.216754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772682.280749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772682.344763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772682.344775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772682.344796:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772682.408782:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772682.472769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772682.472792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772682.472796:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772682.600797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772682.600797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772682.600826:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772682.600849:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772682.664771:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772682.664776:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772682.664808:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772682.664808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772682.664856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772682.664858:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772682.792753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772682.856748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772682.920742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772682.984793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772682.984840:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772682.984842:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772682.984850:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772683.048793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772683.048811:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772683.049069:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772683.176772:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772683.176787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772683.240746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772683.304851:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772683.368771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772683.368777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772683.368788:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772683.432770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772683.496783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772683.496796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772683.496813:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772683.624833:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772683.624932:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772683.624981:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772683.625758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772683.688770:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772683.688784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772683.688809:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772683.688809:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772683.688843:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772683.688853:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772683.816753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772683.880761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772683.944745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772684.008775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772684.008801:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772684.008819:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772684.008852:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772684.072796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772684.072834:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772684.072883:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772684.200747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772684.200793:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772684.264771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772684.328790:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772684.392756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772684.392779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772684.392787:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772684.456773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772684.520774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772684.520792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772684.520810:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772684.648767:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772684.648791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772684.648798:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772684.648800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772684.712712:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772684.712741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772684.712742:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772684.712768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772684.712788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772684.712789:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772684.840715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772684.904732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772684.968754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772685.032780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772685.032787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772685.032792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772685.032813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772685.096795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772685.096802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772685.096811:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772685.224763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772685.224787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772685.288776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772685.352764:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772685.416745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772685.416774:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772685.416789:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772685.480742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772685.544763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772685.544792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772685.544793:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772685.672772:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772685.672811:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772685.672821:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772685.672836:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772685.736744:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772685.736750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772685.736753:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772685.736757:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772685.736784:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772685.736805:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772685.864754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772685.928755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772685.992759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772685.992833:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772685.992865:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772685.992873:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772685.992889:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1681 00000100:00080000:0.0:1790772685.992907:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772685.992926:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772685.992993:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534506880/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772685.993036:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772685.993042:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772685.993045:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772685.993050:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1681 00000100:00080000:0.0:1790772685.993067:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772685.993077:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772685.993094:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772685.993102:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534506880:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772685.993122:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534507008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772685.993141:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772685.993146:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772685.993149:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772685.993156:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1681 00000100:00080000:0.0:1790772685.993161:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772685.993167:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772685.993174:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772685.993233:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534507136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772685.993255:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772685.993258:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772685.993270:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772685.993276:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772685.993288:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1681 00000100:00080000:0.0:1790772685.993303:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772685.993313:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772685.993324:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772685.993340:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772685/real 1790772685] req@ffff9c4ec7a7df80 x1877759534506880/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772685.993358:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534507264/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772685.993377:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534506880/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772685.993389:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772685.993402:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772685.993403:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772685.993415:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772685.993418:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772685.993431:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534506880:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772685.993438:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1681 00000100:00080000:0.0:1790772685.993451:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772685.993465:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772685.993477:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534507008:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790772685.993478:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772685.993520:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534507392/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772685.993531:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772685.993552:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772685.993558:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772685.993561:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772685.993568:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1681 00000100:00080000:0.0:1790772685.993576:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772685.993578:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772685/real 1790772685] req@ffff9c4ec7a7f480 x1877759534507008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772685.993590:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772685.993607:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772685.993615:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534507008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772685.993632:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534507520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772685.993644:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772685.993648:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772685.993693:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772685.993702:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534507008:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772685.993719:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1790772685.993726:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534507136:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772685.993728:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772685.993745:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1681 00000100:00080000:0.0:1790772685.993752:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772685.993762:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772685.993775:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790772685.993776:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772685.993799:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772685/real 1790772685] req@ffff9c4ec7a7c380 x1877759534507136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772685.993813:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534507648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772685.993833:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534507136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772685.993846:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772685.993867:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772685.993874:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772685.993875:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772685.993895:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772685.993896:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1681 00000100:00080000:0.0:1790772685.993910:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772685.993917:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534507136:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772685.993926:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772685.993939:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772685.993940:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534507264:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772685.993971:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534507776/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772685.993987:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772685.993996:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772685.993999:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772685.994005:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772685.994015:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1681 00000100:00080000:0.0:1790772685.994022:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772685.994025:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772685/real 1790772685] req@ffff9c4ec7a7d880 x1877759534507264/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772685.994033:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772685.994047:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772685.994059:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534507264/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772685.994071:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534507904/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772685.994081:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772685.994096:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772685.994105:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534507264:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772685.994135:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534507392:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772685.994162:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772685.994176:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772685/real 1790772685] req@ffff9c4ec7a7ed80 x1877759534507392/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772685.994221:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534507392/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772685.994236:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772685.994241:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772685.994246:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534507392:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772685.994254:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534507520:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772685.994283:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772685.994298:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772685/real 1790772685] req@ffff9c4ec7a7ca80 x1877759534507520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772685.994313:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534507520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772685.994335:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772685.994339:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772685.994343:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534507520:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772685.994351:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534507648:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772685.994371:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772685.994384:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772685/real 1790772685] req@ffff9c4ec7a7e300 x1877759534507648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772685.994399:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534507648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772685.994410:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772685.994414:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772685.994420:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534507648:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772685.994428:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534507776:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772685.994452:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772685.994466:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772685/real 1790772685] req@ffff9c4ec7a7fb80 x1877759534507776/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772685.994483:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534507776/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772685.994498:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772685.994503:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772685.994507:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534507776:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772685.994515:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534507904:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772685.994536:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772685.994551:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772685/real 1790772685] req@ffff9c4ec7a7c700 x1877759534507904/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772685.994568:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534507904/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772685.994580:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772685.994584:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772685.994589:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534507904:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772686.056785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772686.056789:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772686.056791:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772686.056802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772686.120749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772686.120818:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772686.121010:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772686.248759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772686.248784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772686.312757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772686.376793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772686.440765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772686.440794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772686.440799:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772686.504751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772686.568767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772686.568795:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772686.568799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772686.696932:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772686.696932:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772686.697045:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772686.697734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772686.760766:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772686.760823:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772686.760823:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772686.760835:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772686.760842:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772686.760907:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772686.888791:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772686.952774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772687.016729:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772687.080795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772687.080796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772687.080817:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772687.080864:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772687.144868:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772687.144871:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772687.144995:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772687.272787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772687.272833:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772687.336769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772687.400775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772687.464776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772687.464787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772687.464794:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772687.528745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772687.592771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772687.592777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772687.592811:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772687.720776:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772687.720813:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772687.720826:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772687.720855:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772687.784804:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772687.784804:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772687.784805:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772687.784827:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772687.784869:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772687.784890:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772687.912778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772687.976771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772688.040770:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772688.104798:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772688.104798:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772688.104815:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772688.104825:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772688.168750:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772688.168850:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772688.168931:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772688.296756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772688.296813:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772688.360774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772688.424780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772688.488770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772688.488798:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772688.488800:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772688.552765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772688.616770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772688.616773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772688.616811:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772688.744782:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772688.744787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772688.744808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772688.744905:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772688.808755:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772688.808773:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772688.808810:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772688.808887:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772688.808905:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772688.808925:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772688.936771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772689.000750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772689.064760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772689.128752:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772689.128782:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772689.128786:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772689.128787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772689.192775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772689.192876:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772689.192904:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772689.320765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772689.320786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772689.384744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772689.448779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772689.512764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772689.512772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772689.512807:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772689.576763:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772689.640770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772689.640811:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772689.640811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772689.768787:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772689.768822:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772689.768843:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772689.768848:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772689.832773:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772689.832792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772689.832833:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772689.832862:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772689.832881:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772689.832897:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772689.960755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772690.024777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772690.088771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772690.152797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772690.152803:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772690.152804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772690.152810:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772690.216795:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772690.216861:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772690.216976:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772690.344769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772690.344794:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772690.408776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772690.472893:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772690.536766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772690.536766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772690.536833:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772690.600755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772690.664768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772690.664827:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772690.664828:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772690.792788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772690.792788:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772690.792798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772690.792840:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772690.856802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772690.856805:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772690.856876:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772690.856883:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772690.856883:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772690.856885:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772690.984775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772691.048741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772691.112788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772691.112797:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772691.112808:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772691.112816:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772691.112834:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1687 00000100:00080000:0.0:1790772691.112852:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772691.112872:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772691.112939:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534508032/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772691.112983:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772691.112989:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772691.112992:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772691.112998:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1687 00000100:00080000:0.0:1790772691.113003:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772691.113009:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772691.113026:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772691.113046:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534508160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772691.113065:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772691.113066:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534508032:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772691.113103:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772691.113110:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772691.113118:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1687 00000100:00080000:0.0:1790772691.113123:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772691.113129:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772691.113137:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772691.113156:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534508288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772691.113170:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772691.113176:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772691.113181:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772691.113186:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1687 00000100:00080000:0.0:1790772691.113193:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772691.113213:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772691.113221:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:2.0:1790772691.113225:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772691.113243:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534508416/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772691.113259:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772691.113265:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772691.113268:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772691.113274:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1687 00000100:00080000:0.0:1790772691.113279:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772691.113285:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772691.113291:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772691.113309:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534508544/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772691.113325:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772691/real 1790772691] req@ffff9c4ec7a7d180 x1877759534508032/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772691.113326:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772691.113339:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772691.113344:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772691.113359:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1687 00000100:00080000:0.0:1790772691.113370:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772691.113377:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772691.113384:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772691.113395:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534508032/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772691.113404:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534508672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772691.113455:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772691.113459:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772691.113461:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772691.113469:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772691.113483:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1687 00000100:00080000:2.0:1790772691.113483:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772691.113495:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772691.113501:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534508032:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772691.113509:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772691.113519:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772691.113533:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534508800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772691.113545:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772691.113551:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772691.113556:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772691.113558:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534508160:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772691.113567:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1687 00000100:00080000:0.0:1790772691.113575:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772691.113583:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772691.113590:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772691.113609:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534508928/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772691.113622:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772691.113639:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772691.113646:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772691.113682:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772691.113692:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1687 00000100:00080000:0.0:1790772691.113698:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772691.113704:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772691/real 1790772691] req@ffff9c4ec7a7c700 x1877759534508160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772691.113711:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772691.113725:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772691.113748:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534509056/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772691.113759:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534508160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772691.113816:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772691.113822:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772691.113828:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534508160:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772691.113838:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534508288:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772691.113868:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772691.113884:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772691/real 1790772691] req@ffff9c4ec7a7fb80 x1877759534508288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772691.113900:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534508288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772691.113912:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772691.113917:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772691.113922:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534508288:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772691.113929:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534508416:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772691.113955:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772691.113977:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772691/real 1790772691] req@ffff9c4ec7a7e300 x1877759534508416/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772691.114002:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534508416/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772691.114037:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772691.114045:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772691.114053:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534508416:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772691.114064:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534508544:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772691.114124:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772691.114142:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772691/real 1790772691] req@ffff9c4ec7a7ca80 x1877759534508544/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772691.114158:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534508544/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772691.114172:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772691.114177:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772691.114181:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534508544:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772691.114190:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534508672:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772691.114214:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772691.114227:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772691/real 1790772691] req@ffff9c4ec7a7ed80 x1877759534508672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772691.114243:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534508672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772691.114268:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772691.114274:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772691.114283:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534508672:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772691.114296:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534508800:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772691.114338:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772691.114363:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772691/real 1790772691] req@ffff9c4ec7a7d880 x1877759534508800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772691.114380:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534508800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772691.114394:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772691.114401:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772691.114409:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534508800:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772691.114456:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534508928:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772691.114501:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772691.114521:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772691/real 1790772691] req@ffff9c4ec7a7c380 x1877759534508928/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772691.114537:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534508928/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772691.114553:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772691.114557:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772691.114562:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534508928:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772691.114571:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534509056:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772691.114594:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772691.114608:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772691/real 1790772691] req@ffff9c4ec7a7f480 x1877759534509056/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772691.114623:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534509056/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772691.114636:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772691.114639:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772691.114644:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534509056:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772691.176785:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772691.176799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772691.176804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772691.176839:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772691.240790:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772691.240892:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772691.240973:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772691.368748:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772691.368784:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772691.432761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772691.496803:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772691.560784:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772691.560872:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772691.560878:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772691.624754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772691.688767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772691.688777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772691.688803:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772691.816773:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772691.816793:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772691.816793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772691.816815:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772691.880753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772691.880800:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772691.880836:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772691.880844:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772691.880850:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772691.880894:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772692.008742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772692.072808:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772692.136746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772692.200766:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772692.200786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772692.200802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772692.200809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772692.264797:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772692.264840:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772692.264942:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772692.392779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772692.392803:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772692.456748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772692.520787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772692.584773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772692.584806:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772692.584819:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772692.648766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772692.712774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772692.712839:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772692.712843:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772692.840795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772692.840811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772692.840812:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772692.840828:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772692.904800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772692.904811:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772692.904827:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772692.904832:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772692.904871:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772692.904886:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772693.032746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772693.096755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772693.160741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772693.224793:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772693.224806:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772693.224807:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772693.224809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772693.288759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772693.288829:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772693.288996:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772693.416736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772693.416777:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772693.480755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772693.544768:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772693.608757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772693.608788:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772693.609784:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772693.672760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772693.736787:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772693.736830:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772693.736873:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772693.864839:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772693.864847:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772693.864853:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772693.864878:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772693.928801:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772693.928840:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772693.928861:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772693.928864:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772693.928875:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772693.928915:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772694.056752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772694.120765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772694.184785:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772694.248853:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772694.248855:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772694.248866:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772694.248871:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772694.312863:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772694.312923:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772694.312930:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772694.440745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772694.440780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772694.504764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772694.568788:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772694.632722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772694.632835:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772694.632838:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772694.696768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772694.760753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772694.760801:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772694.760801:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772694.888756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772694.888818:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772694.888837:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772694.888848:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772694.952721:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772694.952734:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772694.952751:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772694.952762:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772694.952781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772694.952801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772695.080717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772695.144711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772695.208721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772695.272714:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772695.272734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772695.272739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772695.272741:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772695.336788:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772695.336848:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772695.336930:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772695.464712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772695.464726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772695.528710:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772695.592744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772695.656695:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772695.656712:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772695.656736:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772695.720720:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772695.784735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772695.784735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772695.784735:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772695.912738:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772695.912742:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772695.912747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772695.912782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772695.976702:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772695.976750:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772695.976808:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772695.976834:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772695.976835:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772695.976857:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772696.104720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772696.168748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790772696.232791:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772696.232802:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1790772696.232810:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772696.232810:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772696.232829:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1692 00000100:00080000:2.0:1790772696.232846:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772696.232865:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772696.232925:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534509184/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772696.232968:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772696.232974:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772696.232977:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772696.233025:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1692 00000100:00080000:2.0:1790772696.233032:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772696.233040:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772696.233057:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772696.233079:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534509312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772696.233097:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772696.233102:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772696.233106:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772696.233113:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534509184:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772696.233117:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1692 00000100:00080000:2.0:1790772696.233127:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772696.233141:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772696.233151:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772696.233181:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534509440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772696.233202:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772696.233211:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772696.233215:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772696.233224:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1692 00000100:00080000:2.0:1790772696.233233:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772696.233248:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:3.0:1790772696.233255:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790772696.233265:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772696.233294:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534509568/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772696.233308:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772696.233313:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772696.233316:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772696.233324:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1692 00000100:00080000:2.0:1790772696.233331:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772696.233339:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772696.233348:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772696.233350:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772696/real 1790772696] req@ffff9c4efbcc2a00 x1877759534509184/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772696.233370:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534509696/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772696.233385:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772696.233394:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534509184/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772696.233396:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772696.233403:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772696.233419:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1692 00000100:00080000:2.0:1790772696.233428:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772696.233438:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772696.233439:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772696.233455:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1790772696.233456:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772696.233471:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534509184:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772696.233482:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534509824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772696.233496:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772696.233501:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772696.233505:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772696.233512:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1692 00000100:00080000:2.0:1790772696.233517:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772696.233521:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534509312:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772696.233529:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772696.233539:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772696.233558:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534509952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772696.233575:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1790772696.233583:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772696.233585:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772696.233592:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772696.233602:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1692 00000100:00080000:2.0:1790772696.233609:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772696.233616:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772696/real 1790772696] req@ffff9c4efbcc1500 x1877759534509312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772696.233620:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772696.233634:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772696.233700:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534509312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772696.233713:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534510080/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772696.233732:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772696.233737:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772696.233738:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772696.233746:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772696.233753:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772696.233756:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1692 00000100:00080000:2.0:1790772696.233765:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772696.233768:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534509312:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772696.233777:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772696.233788:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534509440:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790772696.233789:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772696.233815:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534510208/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772696.233838:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772696.233864:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772696/real 1790772696] req@ffff9c4efbcc1c00 x1877759534509440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772696.233888:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534509440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772696.233901:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772696.233906:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772696.233910:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534509440:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772696.233919:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534509568:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772696.233941:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772696.233953:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772696/real 1790772696] req@ffff9c4efbcc1880 x1877759534509568/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772696.233968:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534509568/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772696.234017:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772696.234025:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772696.234033:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534509568:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772696.234047:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534509696:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772696.234092:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772696.234115:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772696/real 1790772696] req@ffff9c4efbcc2300 x1877759534509696/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772696.234142:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534509696/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772696.234155:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772696.234160:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772696.234165:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534509696:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772696.234173:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534509824:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772696.234199:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772696.234212:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772696/real 1790772696] req@ffff9c4efbcc2680 x1877759534509824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772696.234235:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534509824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772696.234264:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772696.234271:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772696.234280:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534509824:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772696.234325:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534509952:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772696.234359:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772696.234375:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772696/real 1790772696] req@ffff9c4efbcc0700 x1877759534509952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772696.234391:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534509952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772696.234411:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772696.234416:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772696.234421:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534509952:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772696.234429:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534510080:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772696.234449:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772696.234461:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772696/real 1790772696] req@ffff9c4efbcc1f80 x1877759534510080/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772696.234476:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534510080/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772696.234490:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772696.234494:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772696.234498:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534510080:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772696.234505:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534510208:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772696.234525:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772696.234536:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772696/real 1790772696] req@ffff9c4efbcc0a80 x1877759534510208/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772696.234551:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534510208/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772696.234563:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772696.234567:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772696.234572:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534510208:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772696.296785:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772696.296793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772696.296826:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772696.296835:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772696.360803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772696.360839:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772696.360894:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772696.488752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772696.488795:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772696.552747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772696.616757:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772696.680759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772696.680766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772696.680787:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772696.744778:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772696.808718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772696.808763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772696.808782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772696.936810:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772696.936823:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772696.936831:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772696.936832:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772697.000790:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772697.000792:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772697.000815:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772697.000860:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772697.000875:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772697.000894:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772697.128752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772697.192773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772697.256755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772697.320769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772697.320793:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772697.320828:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772697.320866:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772697.384807:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772697.384848:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772697.385007:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772697.512786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772697.512811:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772697.576764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772697.640898:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772697.704761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772697.704775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772697.704813:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772697.768766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772697.832771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772697.832784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772697.832797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772697.960771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772697.960785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772697.960811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772697.960859:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772698.024789:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772698.024819:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772698.024828:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772698.024866:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772698.024883:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772698.024910:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772698.152758:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772698.216743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772698.280753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772698.344831:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772698.344845:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772698.344847:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772698.344853:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772698.408760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772698.408860:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772698.408988:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772698.536758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772698.536802:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772698.600779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772698.664778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772698.728800:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772698.728814:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772698.728916:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772698.792779:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772698.856754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772698.856781:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772698.856792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772698.984764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772698.984847:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772698.984849:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772698.984850:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772699.048787:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772699.048793:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772699.048833:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772699.048838:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772699.048882:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772699.048888:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772699.176758:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772699.240746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772699.304773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772699.368808:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772699.368841:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772699.368864:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772699.368865:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772699.432796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772699.432874:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772699.432890:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772699.560724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772699.560813:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772699.624774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772699.688793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772699.752774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772699.752779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772699.752828:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772699.816762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772699.880745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772699.880772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772699.880834:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772700.008754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772700.008788:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772700.008803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772700.008837:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772700.072774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772700.072786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772700.072830:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772700.072887:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772700.072893:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772700.072934:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772700.200753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772700.264750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772700.328739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772700.392775:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772700.392782:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772700.392852:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772700.392855:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772700.456754:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772700.456853:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772700.457060:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772700.584769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772700.584799:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772700.648756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772700.712817:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772700.776756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772700.776767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772700.776805:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772700.840845:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772700.904761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772700.904793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772700.904817:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772701.032774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772701.032775:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772701.032785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772701.032846:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772701.096836:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772701.096852:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772701.096877:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772701.096922:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772701.096946:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772701.096977:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772701.224773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772701.288765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772701.352782:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772701.352789:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772701.352802:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772701.352809:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772701.352840:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1697 00000100:00080000:2.0:1790772701.352860:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772701.352912:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772701.352991:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534510336/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772701.353043:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772701.353049:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772701.353052:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772701.353059:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1697 00000100:00080000:2.0:1790772701.353066:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772701.353075:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772701.353093:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772701.353110:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534510336:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772701.353122:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534510464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772701.353148:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772701.353155:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772701.353160:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772701.353172:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1697 00000100:00080000:2.0:1790772701.353182:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772701.353213:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772701.353227:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1790772701.353231:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772701.353261:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534510592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772701.353277:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772701.353282:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772701.353285:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772701.353294:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1697 00000100:00080000:2.0:1790772701.353301:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772701.353308:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772701/real 1790772701] req@ffff9c4efbcc0380 x1877759534510336/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772701.353317:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772701.353330:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772701.353354:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534510720/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772701.353371:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772701.353374:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534510336/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772701.353379:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772701.353385:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772701.353400:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1697 00000100:00080000:2.0:1790772701.353410:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772701.353422:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772701.353429:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:3.0:1790772701.353433:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772701.353446:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772701.353449:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534510848/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772701.353459:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534510336:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772701.353486:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772701.353492:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772701.353495:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772701.353502:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1697 00000100:00080000:2.0:1790772701.353506:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772701.353512:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772701.353533:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772701.353546:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534510464:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772701.353553:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534510976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772701.353574:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772701.353579:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772701.353582:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772701.353589:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1697 00000100:00080000:2.0:1790772701.353606:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1790772701.353615:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772701.353620:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772701.353631:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772701.353687:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534511104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772701.353697:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772701/real 1790772701] req@ffff9c4efbcc3100 x1877759534510464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772701.353703:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772701.353717:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772701.353724:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772701.353734:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1697 00000100:00080000:2.0:1790772701.353743:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772701.353744:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534510464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772701.353758:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772701.353774:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:3.0:1790772701.353796:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1790772701.353798:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534511232/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772701.353812:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772701.353830:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772701.353831:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534510464:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772701.353847:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772701.353853:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772701.353858:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534510592:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772701.353865:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1697 00000100:00080000:2.0:1790772701.353907:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772701.353922:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772701.353935:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:3.0:1790772701.353947:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772701.353968:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534511360/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772701.353976:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772701/real 1790772701] req@ffff9c4efbcc0a80 x1877759534510592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772701.354035:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534510592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772701.354050:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772701.354057:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772701.354063:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534510592:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772701.354073:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534510720:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772701.354098:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772701.354111:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772701/real 1790772701] req@ffff9c4efbcc1f80 x1877759534510720/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772701.354126:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534510720/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772701.354141:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772701.354145:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772701.354150:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534510720:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772701.354157:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534510848:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772701.354176:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772701.354188:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772701/real 1790772701] req@ffff9c4efbcc0700 x1877759534510848/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772701.354203:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534510848/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772701.354216:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772701.354220:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772701.354224:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534510848:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772701.354255:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534510976:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772701.354277:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772701.354290:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772701/real 1790772701] req@ffff9c4efbcc2680 x1877759534510976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772701.354306:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534510976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772701.354332:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772701.354340:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772701.354349:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534510976:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772701.354362:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534511104:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772701.354402:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772701.354427:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772701/real 1790772701] req@ffff9c4efbcc2300 x1877759534511104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772701.354445:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534511104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772701.354457:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772701.354462:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772701.354467:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534511104:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772701.354478:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534511232:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772701.354518:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772701.354542:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772701/real 1790772701] req@ffff9c4efbcc1880 x1877759534511232/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772701.354570:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534511232/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772701.354591:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772701.354596:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772701.354601:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534511232:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772701.354610:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534511360:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772701.354639:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772701.354686:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772701/real 1790772701] req@ffff9c4efbcc1c00 x1877759534511360/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772701.354704:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534511360/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772701.354718:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772701.354722:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772701.354726:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534511360:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772701.416758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772701.416785:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772701.416812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772701.416850:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772701.480755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772701.480845:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772701.480923:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772701.608762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772701.608775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772701.672789:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772701.736751:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772701.800819:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772701.800819:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772701.800823:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772701.864754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772701.928806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772701.928835:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772701.928843:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772702.056785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772702.056798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772702.056892:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772702.056899:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772702.120798:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772702.120801:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772702.120810:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772702.120862:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772702.120918:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772702.120939:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772702.249799:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772702.312770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772702.376756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772702.440790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772702.440793:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772702.440798:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772702.440812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772702.504795:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772702.504813:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772702.505038:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772702.632799:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772702.632819:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772702.696790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772702.760795:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772702.824761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772702.824799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772702.824805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772702.888773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772702.952775:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772702.952779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772702.952801:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772703.080773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772703.080798:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772703.080799:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772703.080823:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772703.144791:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772703.144793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772703.144816:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772703.144868:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772703.144892:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772703.144900:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772703.272778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772703.336786:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772703.400762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772703.464805:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772703.464924:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772703.464924:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772703.464927:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772703.528891:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772703.528892:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772703.529062:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772703.656765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772703.656829:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772703.720766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772703.784797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772703.848763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772703.848786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772703.848793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772703.912771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772703.976774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772703.976793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772703.976812:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772704.104806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772704.104812:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772704.104820:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772704.104854:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772704.168796:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772704.168805:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772704.168814:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772704.168870:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772704.168932:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772704.168934:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772704.297754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772704.360736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772704.424795:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772704.488776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772704.488868:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772704.488871:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772704.488880:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772704.552872:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772704.552926:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772704.553029:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772704.680749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772704.680808:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772704.744790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772704.808876:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772704.872767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772704.872803:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772704.872836:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772704.936771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772705.000767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772705.000841:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772705.000860:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772705.128766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772705.128792:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772705.128801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772705.128832:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772705.192777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772705.192837:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772705.192849:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772705.192896:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772705.192898:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772705.192900:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772705.321735:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772705.384739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772705.448769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772705.512824:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772705.512835:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772705.512839:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772705.512886:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772705.576779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772705.576907:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772705.576970:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772705.704778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772705.704778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772705.768768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772705.832762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772705.896875:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772705.896889:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772705.896891:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772705.960833:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772706.024758:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772706.024917:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772706.024918:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772706.152827:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772706.152847:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772706.152861:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772706.152878:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772706.216840:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772706.216845:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772706.216856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772706.216858:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772706.216912:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772706.216936:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772706.344751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772706.408817:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772706.472738:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772706.472896:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772706.472911:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772706.472921:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772706.472940:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1702 00000100:00080000:3.0:1790772706.472958:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772706.472977:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772706.473046:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534511488/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772706.473096:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772706.473102:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772706.473105:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772706.473111:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1702 00000100:00080000:3.0:1790772706.473117:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772706.473124:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772706.473140:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772706.473162:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534511616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772706.473173:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534511488:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772706.473184:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772706.473192:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772706.473195:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772706.473202:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1702 00000100:00080000:3.0:1790772706.473207:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772706.473215:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772706.473222:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772706.473238:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534511744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772706.473253:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772706.473258:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772706.473261:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772706.473267:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1702 00000100:00080000:3.0:1790772706.473275:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772706.473281:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772706.473289:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:2.0:1790772706.473299:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772706.473313:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534511872/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772706.473329:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772706.473334:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772706.473337:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772706.473343:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1702 00000100:00080000:3.0:1790772706.473348:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772706.473354:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772706.473361:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772706.473379:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772706/real 1790772706] req@ffff9c4ef73c0e00 x1877759534511488/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772706.473382:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534512000/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772706.473452:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772706.473457:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772706.473461:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772706.473469:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534511488/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772706.473473:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1702 00000100:00080000:3.0:1790772706.473484:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772706.473497:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772706.473505:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790772706.473506:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772706.473523:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772706.473531:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534512128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772706.473539:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534511488:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772706.473564:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772706.473570:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772706.473573:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772706.473581:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1702 00000100:00080000:3.0:1790772706.473586:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772706.473596:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534511616:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772706.473598:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772706.473613:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772706.473633:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534512256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772706.473649:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772706.473690:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772706.473708:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772706.473712:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772706.473720:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772706/real 1790772706] req@ffff9c4ef73c2d80 x1877759534511616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772706.473728:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1702 00000100:00080000:3.0:1790772706.473741:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772706.473754:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772706.473760:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534511616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772706.473766:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772706.473828:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534512384/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772706.473838:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772706.473859:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772706.473862:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772706.473876:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534511616:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772706.473878:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772706.473887:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772706.473905:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1702 00000100:00100000:2.0:1790772706.473907:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534511744:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772706.473921:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772706.473936:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772706.473947:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:2.0:1790772706.473963:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772706.473974:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534512512/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772706.473990:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772706/real 1790772706] req@ffff9c4ef73c1500 x1877759534511744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772706.474011:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534511744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772706.474029:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772706.474038:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772706.474045:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534511744:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772706.474058:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534511872:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772706.474103:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772706.474129:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772706/real 1790772706] req@ffff9c4ef73c0a80 x1877759534511872/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772706.474151:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534511872/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772706.474166:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772706.474171:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772706.474176:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534511872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772706.474184:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534512000:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772706.474213:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772706.474227:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772706/real 1790772706] req@ffff9c4ef73c3800 x1877759534512000/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772706.474242:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534512000/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772706.474255:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772706.474259:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772706.474263:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534512000:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772706.474271:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534512128:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772706.474292:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772706.474305:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772706/real 1790772706] req@ffff9c4ef73c2300 x1877759534512128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772706.474330:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534512128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772706.474350:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772706.474354:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772706.474358:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534512128:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772706.474392:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534512256:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772706.474415:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772706.474428:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772706/real 1790772706] req@ffff9c4ef73c0700 x1877759534512256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772706.474443:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534512256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772706.474455:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772706.474458:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772706.474463:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534512256:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772706.474470:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534512384:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772706.474490:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772706.474501:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772706/real 1790772706] req@ffff9c4ef73c2a00 x1877759534512384/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772706.474516:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534512384/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772706.474539:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772706.474544:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772706.474549:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534512384:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772706.474555:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534512512:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772706.474574:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772706.474586:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772706/real 1790772706] req@ffff9c4ef73c1c00 x1877759534512512/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772706.474601:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534512512/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772706.474614:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772706.474618:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772706.474622:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534512512:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772706.536848:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772706.536848:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772706.536855:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772706.536861:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772706.600871:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772706.600888:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772706.601108:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772706.728758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772706.728764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772706.792748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772706.856762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772706.920792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772706.920824:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772706.920834:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772706.984749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772707.048816:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772707.048834:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772707.048859:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772707.176828:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772707.176894:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772707.176900:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772707.176903:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772707.240799:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772707.240831:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772707.240852:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772707.240870:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772707.240898:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772707.240918:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772707.368747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772707.432749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772707.496790:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772707.560798:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772707.560799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772707.560812:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772707.560815:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772707.624820:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772707.624876:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772707.625097:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772707.752814:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772707.752828:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772707.816806:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772707.880787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772707.944795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772707.944859:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772707.944876:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772708.008784:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772708.072791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772708.072804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772708.072849:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772708.200782:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772708.200814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772708.200863:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772708.200864:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772708.264807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772708.264838:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772708.264858:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772708.264905:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772708.264909:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772708.264912:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772708.392772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772708.456781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772708.520800:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772708.584814:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772708.584819:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772708.584820:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772708.584839:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772708.648807:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772708.648859:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772708.649035:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772708.776775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772708.776783:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772708.840778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772708.904808:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772708.968795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772708.968819:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772708.968862:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772709.032851:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772709.096811:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772709.096811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772709.096825:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772709.224840:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772709.224841:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772709.224849:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772709.224855:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772709.288817:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772709.288838:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772709.288842:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772709.288851:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772709.288892:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772709.288915:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772709.416795:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772709.480811:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772709.544754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772709.608809:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772709.608813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772709.608832:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772709.608834:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772709.672790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772709.672856:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772709.672901:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772709.800816:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772709.800835:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772709.864777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772709.928831:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772709.992790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772709.992799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772709.992824:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772710.056762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772710.120774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772710.120818:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772710.120845:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772710.248778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772710.248800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772710.248805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772710.248853:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772710.312790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772710.312814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772710.312855:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772710.312872:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772710.312885:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772710.312886:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772710.440771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772710.504763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772710.568795:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772710.632793:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772710.632793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772710.632794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772710.632797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772710.696838:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772710.696852:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772710.696872:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772710.824769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772710.824769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772710.888788:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772710.952778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772711.016759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772711.016811:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772711.016811:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772711.080823:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772711.144822:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772711.144826:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772711.144826:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772711.272742:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772711.272742:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772711.272742:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772711.272798:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772711.336750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772711.336773:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772711.336773:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772711.336819:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772711.336820:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772711.336822:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772711.464839:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772711.528781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772711.592747:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772711.592748:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772711.592758:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772711.592762:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772711.592771:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1707 00000100:00080000:3.0:1790772711.592781:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772711.592792:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772711.592826:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534512640/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772711.592853:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772711.592857:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772711.592858:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772711.592862:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1707 00000100:00080000:3.0:1790772711.592864:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772711.592867:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772711.592875:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772711.592885:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534512768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772711.592893:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772711.592896:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772711.592897:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772711.592900:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1707 00000100:00080000:3.0:1790772711.592903:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772711.592906:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772711.592911:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772711.592914:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534512640:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772711.592921:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534512896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772711.592929:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772711.592932:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772711.592934:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772711.592936:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1707 00000100:00080000:3.0:1790772711.592939:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772711.592942:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772711.592946:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772711.592954:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534513024/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772711.592962:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772711.592965:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772711.592966:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772711.592969:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1707 00000100:00080000:3.0:1790772711.592972:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772711.592976:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772711.592977:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772711.592983:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772711.592990:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534513152/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772711.592998:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772711.593001:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772711.593002:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772711.593005:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1707 00000100:00080000:3.0:1790772711.593008:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772711.593012:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772711.593016:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772711.593019:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772711/real 1790772711] req@ffff9c4ef73c0000 x1877759534512640/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772711.593027:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534513280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772711.593039:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534512640/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772711.593044:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772711.593050:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772711.593054:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772711.593059:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772711.593059:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1707 00000100:00080000:3.0:1790772711.593061:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772711.593066:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772711.593068:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772711.593073:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534512640:192.168.204.150@tcp:250:kworker.0 00000008:00100000:3.0:1790772711.593073:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772711.593083:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534512768:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772711.593087:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534513408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772711.593095:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772711.593098:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772711.593100:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790772711.593104:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772711.593107:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1707 00000100:00080000:3.0:1790772711.593111:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772711.593116:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772711.593119:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772711/real 1790772711] req@ffff9c4ef73c1c00 x1877759534512768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772711.593121:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772711.593137:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534512768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772711.593137:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534513536/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772711.593166:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772711.593169:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772711.593171:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772711.593174:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772711.593175:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1707 00000100:00080000:3.0:1790772711.593181:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772711.593182:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772711.593187:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772711.593188:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534512768:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772711.593194:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772711.593204:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534513664/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772711.593222:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534512896:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772711.593238:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772711.593245:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772711/real 1790772711] req@ffff9c4ef73c2a00 x1877759534512896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772711.593253:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534512896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772711.593260:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772711.593262:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772711.593265:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534512896:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772711.593269:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534513024:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772711.593279:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772711.593285:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772711/real 1790772711] req@ffff9c4ef73c0700 x1877759534513024/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772711.593293:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534513024/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772711.593300:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772711.593302:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772711.593304:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534513024:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772711.593309:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534513152:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772711.593327:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772711.593339:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772711/real 1790772711] req@ffff9c4ef73c2300 x1877759534513152/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772711.593352:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534513152/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772711.593363:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772711.593367:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772711.593371:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534513152:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772711.593377:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534513280:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772711.593392:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772711.593400:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772711/real 1790772711] req@ffff9c4ef73c3800 x1877759534513280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772711.593408:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534513280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772711.593424:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772711.593426:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772711.593429:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534513280:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772711.593432:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534513408:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772711.593443:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772711.593449:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772711/real 1790772711] req@ffff9c4ef73c0a80 x1877759534513408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772711.593456:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534513408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772711.593462:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772711.593464:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772711.593466:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534513408:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772711.593469:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534513536:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772711.593480:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772711.593485:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772711/real 1790772711] req@ffff9c4ef73c1500 x1877759534513536/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772711.593493:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534513536/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772711.593499:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772711.593501:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772711.593504:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534513536:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772711.593508:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534513664:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772711.593518:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772711.593524:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772711/real 1790772711] req@ffff9c4ef73c2d80 x1877759534513664/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772711.593531:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534513664/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772711.593537:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772711.593539:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772711.593541:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534513664:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772711.656748:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772711.656750:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772711.656753:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772711.656754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772711.720746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772711.720751:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772711.720826:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772711.848731:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772711.848734:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772711.912735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772711.976809:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772712.040750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772712.040750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772712.040751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772712.104729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772712.168737:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772712.168738:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772712.168743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772712.296770:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772712.296770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772712.296770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772712.296770:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772712.360725:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772712.360733:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772712.360745:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772712.360752:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772712.360771:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772712.360777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772712.488743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772712.552755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772712.616728:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772712.680779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772712.680779:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772712.680779:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772712.680800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772712.744757:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772712.744786:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772712.744810:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772712.872737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772712.872787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772712.936766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772713.000771:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772713.064759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772713.064759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772713.064813:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772713.128813:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772713.192785:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772713.192798:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772713.192799:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772713.320799:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772713.320806:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772713.320828:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772713.320832:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772713.384788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772713.384793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772713.384802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772713.384808:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772713.384846:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772713.384861:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772713.512743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772713.576763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772713.640774:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772713.704724:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772713.704765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772713.704781:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772713.704800:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772713.768729:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772713.768794:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772713.768869:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772713.896729:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772713.896736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772713.960719:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772714.024755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772714.088755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772714.088766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772714.088772:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772714.152717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772714.216752:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772714.216753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772714.216753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772714.344747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772714.344747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772714.344750:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772714.344750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772714.408749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772714.408755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772714.408782:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772714.408785:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772714.408840:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772714.408841:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772714.536732:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772714.600719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772714.664712:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772714.728778:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772714.728785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772714.728797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772714.728829:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772714.792723:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772714.792792:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772714.792851:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772714.920781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772714.920782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772714.984747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772715.048714:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772715.112733:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772715.112737:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772715.112750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772715.176718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772715.240748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772715.240760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772715.240760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772715.368766:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772715.368800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772715.368801:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772715.368801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772715.432726:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772715.432766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772715.432783:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772715.432791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772715.432818:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772715.432822:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772715.560739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772715.624728:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772715.688716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772715.752748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772715.752795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772715.752807:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772715.752824:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772715.816811:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772715.816862:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772715.816914:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772715.944747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772715.944751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772716.008723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772716.072778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772716.136775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772716.136787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772716.136788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772716.200719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772716.264764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772716.264794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772716.264794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772716.392726:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772716.392747:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772716.392787:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772716.392794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772716.456785:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772716.456787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772716.456796:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772716.456806:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772716.456858:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772716.456859:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772716.584752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772716.648729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772716.712768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772716.712822:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772716.712836:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772716.712845:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772716.712865:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1712 00000100:00080000:1.0:1790772716.712885:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772716.712908:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772716.712982:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534513792/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772716.713039:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772716.713049:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772716.713054:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772716.713063:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1712 00000100:00080000:1.0:1790772716.713072:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772716.713087:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772716.713108:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772716.713121:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534513792:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772716.713144:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534513920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772716.713163:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772716.713169:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772716.713173:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772716.713180:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1712 00000100:00080000:1.0:1790772716.713186:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772716.713195:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772716.713203:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772716.713222:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534514048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772716.713240:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772716.713247:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772716.713254:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772716.713257:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772716.713265:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1712 00000100:00080000:1.0:1790772716.713270:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772716.713278:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772716.713286:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772716.713300:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534514176/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772716.713320:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772716.713327:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772716.713336:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772716.713339:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772716/real 1790772716] req@ffff9c4efac4fb80 x1877759534513792/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772716.713352:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1712 00000100:00080000:1.0:1790772716.713369:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772716.713385:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772716.713391:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534513792/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772716.713400:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:2.0:1790772716.713412:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772716.713422:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790772716.713431:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534514304/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772716.713433:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534513792:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772716.713462:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772716.713468:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772716.713471:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772716.713480:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1712 00000100:00080000:1.0:1790772716.713486:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772716.713492:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534513920:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772716.713502:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772716.713512:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772716.713532:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534514432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772716.713548:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772716.713555:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772716.713562:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772716.713570:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772716.713612:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1712 00000100:00100000:2.0:1790772716.713616:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772716/real 1790772716] req@ffff9c4efac4df80 x1877759534513920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772716.713624:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772716.713644:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772716.713696:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534513920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772716.713714:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772716.713735:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790772716.713744:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534514560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772716.713755:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772716.713769:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772716.713775:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534513920:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772716.713778:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772716.713787:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772716.713798:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534514048:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772716.713806:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1712 00000100:00080000:1.0:1790772716.713818:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772716.713830:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772716.713843:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:2.0:1790772716.713865:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772716.713875:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534514688/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772716.713893:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772716/real 1790772716] req@ffff9c4efac4e300 x1877759534514048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772716.713905:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772716.713917:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1790772716.713919:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534514048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772716.713924:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772716.713947:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1712 00000100:00080000:2.0:1790772716.713949:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772716.713962:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772716.713963:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772716.713975:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534514048:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772716.713985:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772716.713994:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534514176:192.168.204.150@tcp:8:kworker.0 00000008:00100000:1.0:1790772716.714000:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772716.714036:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534514816/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772716.714036:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772716.714058:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772716/real 1790772716] req@ffff9c4efac4f480 x1877759534514176/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772716.714090:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534514176/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772716.714105:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772716.714110:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772716.714115:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534514176:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772716.714123:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534514304:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772716.714145:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772716.714157:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772716/real 1790772716] req@ffff9c4efac4f800 x1877759534514304/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772716.714172:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534514304/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772716.714184:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772716.714189:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772716.714193:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534514304:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772716.714222:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534514432:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772716.714260:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772716.714284:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772716/real 1790772716] req@ffff9c4efac4dc00 x1877759534514432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772716.714311:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534514432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772716.714350:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772716.714355:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772716.714360:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534514432:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772716.714370:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534514560:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772716.714401:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772716.714417:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772716/real 1790772716] req@ffff9c4efac4ed80 x1877759534514560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772716.714433:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534514560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772716.714445:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772716.714449:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772716.714453:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534514560:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772716.714461:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534514688:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772716.714481:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772716.714492:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772716/real 1790772716] req@ffff9c4efac4c700 x1877759534514688/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772716.714507:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534514688/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772716.714520:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772716.714524:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772716.714529:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534514688:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772716.714536:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534514816:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772716.714555:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772716.714567:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772716/real 1790772716] req@ffff9c4efac4ea00 x1877759534514816/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772716.714613:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534514816/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772716.714628:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772716.714632:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772716.714637:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534514816:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772716.776749:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772716.776781:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772716.776790:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772716.776835:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772716.840805:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772716.840838:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772716.840961:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772716.968764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772716.968775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772717.032757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772717.096747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772717.160758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772717.160769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772717.160782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772717.224767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772717.288747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772717.288786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772717.288794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772717.416729:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772717.416729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772717.416743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772717.416748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772717.480732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772717.480743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772717.480746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772717.480755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772717.480775:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772717.480784:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772717.608715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772717.672702:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772717.736722:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772717.800713:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772717.800719:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772717.800722:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772717.800723:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772717.864722:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772717.864725:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772717.864796:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772717.992724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772717.992725:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772718.056724:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772718.120818:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772718.184725:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772718.184725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772718.184725:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772718.248708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772718.312710:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772718.312716:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772718.312729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772718.440711:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772718.440712:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772718.440715:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772718.440719:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772718.504728:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772718.504728:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772718.504729:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772718.504733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772718.504761:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772718.504779:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772718.632720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772718.696718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772718.760707:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772718.824718:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772718.824718:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772718.824721:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772718.824725:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772718.888719:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772718.888763:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772718.888802:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772719.016707:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772719.016716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772719.080718:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772719.144800:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772719.208720:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772719.208743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772719.208743:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772719.272703:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772719.336745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772719.336745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772719.336746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772719.464705:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772719.464721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772719.464727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772719.464728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772719.528730:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772719.528735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772719.528745:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772719.528753:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772719.528775:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772719.528779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772719.656706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772719.720717:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772719.784720:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772719.848778:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772719.848782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772719.848787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772719.848793:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772719.912743:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772719.912791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772719.912896:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772720.040729:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772720.040730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772720.104723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772720.168726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772720.232732:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772720.232736:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772720.232742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772720.296724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772720.360708:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772720.360712:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772720.360720:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772720.488722:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772720.488723:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772720.488728:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772720.488728:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772720.552743:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772720.552743:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772720.552743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772720.552743:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772720.552777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772720.552781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772720.680715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772720.744729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772720.808712:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772720.872717:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772720.872718:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772720.872725:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772720.872725:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772720.936732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772720.936748:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772720.936797:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772721.064719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772721.064719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772721.128713:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772721.192730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772721.256718:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772721.256721:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772721.256730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772721.320724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772721.384739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772721.384740:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772721.384740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772721.512719:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772721.512724:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772721.512726:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772721.512727:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772721.576736:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772721.576736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772721.576740:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772721.576758:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772721.576770:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772721.576783:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772721.704705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772721.768730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772721.832717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772721.832734:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772721.832758:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772721.832763:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772721.832775:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1717 00000100:00080000:1.0:1790772721.832789:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772721.832805:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772721.832859:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534514944/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772721.832884:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772721.832887:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772721.832889:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772721.832892:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1717 00000100:00080000:1.0:1790772721.832895:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772721.832898:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772721.832908:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772721.832921:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534515072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772721.832929:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772721.832933:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772721.832934:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772721.832937:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1717 00000100:00080000:1.0:1790772721.832939:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772721.832942:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772721.832946:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772721.832955:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534515200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772721.832979:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772721.832984:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772721.832987:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772721.832988:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534514944:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772721.832992:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1717 00000100:00080000:1.0:1790772721.832998:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772721.833003:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772721.833007:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772721.833016:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534515328/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772721.833023:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772721.833026:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772721.833027:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772721.833030:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1717 00000100:00080000:1.0:1790772721.833032:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772721.833035:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772721.833039:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772721.833048:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534515456/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772721.833054:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772721.833056:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772721.833058:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772721.833061:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1717 00000100:00080000:1.0:1790772721.833071:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772721.833075:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772721.833079:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772721.833088:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534515584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772721.833116:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772721.833122:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772721.833125:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772721.833131:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1717 00000100:00080000:1.0:1790772721.833136:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772721.833144:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772721.833154:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772721.833188:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534515712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772721.833189:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534515072:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772721.833203:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772721.833207:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772721.833209:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772721.833213:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1717 00000100:00080000:1.0:1790772721.833217:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772721.833222:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772721.833227:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790772721.833237:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534515840/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772721.833268:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772721.833276:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534515200:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772721.833277:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772721.833285:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772721.833296:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1717 00000100:00080000:1.0:1790772721.833303:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772721.833311:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772721.833320:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772721.833332:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534515328:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772721.833343:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534515968/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772721.833399:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534515456:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772721.833427:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534515584:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772721.833453:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534515712:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772721.833473:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534515840:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772721.833513:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534515968:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772721.896773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772721.896795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772721.896821:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772721.896845:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772721.960788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772721.960898:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772721.960957:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772722.088775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772722.088784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772722.152791:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772722.216764:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772722.280762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772722.280779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772722.280819:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772722.344785:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772722.408786:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772722.408868:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772722.408869:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772722.536769:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772722.536780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772722.536806:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772722.536820:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772722.600786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772722.600799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772722.600828:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772722.600835:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772722.600865:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772722.600890:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772722.728783:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772722.792739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772722.856891:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772722.920743:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772722.920798:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772722.920811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772722.920824:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772722.984799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772722.984823:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772722.984902:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772723.112799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772723.112814:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772723.176780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772723.240784:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772723.304790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772723.304793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772723.304795:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772723.368755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772723.432790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772723.432794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772723.432800:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772723.560753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772723.560795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772723.560800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772723.560806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772723.624743:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772723.624782:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772723.624804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772723.624813:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772723.624830:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772723.624848:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772723.752772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772723.816787:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772723.880897:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772723.944801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772723.944801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772723.944824:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772723.944830:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772724.008827:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772724.008844:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772724.008917:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772724.136771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772724.136788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772724.200756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772724.264774:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772724.328753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772724.328778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772724.328802:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772724.392697:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772724.456740:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772724.456740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772724.456789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772724.584748:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772724.584748:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772724.584763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772724.584777:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772724.648723:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772724.648730:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772724.648772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772724.648852:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772724.648857:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772724.648905:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772724.776714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772724.840712:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772724.904979:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:2.0:1790772724.905010:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:2.0:1790772724.905065:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1790772724.905088:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790772724.905155:0:2383:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:2.0:1790772724.905185:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:3.0:1790772724.905201:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772721/real 1790772724] req@ffff9c4efac4df80 x1877759534515968/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1790772724.905203:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790772724.905218:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790772724.905231:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:3.0:1790772724.905242:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534515968/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1790772724.905248:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790772724.905264:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:3.0:1790772724.905272:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000800:00000100:2.0:1790772724.905276:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:3.0:1790772724.905284:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772724.905298:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534515968:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772724.905314:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772721/real 1790772724] req@ffff9c4efac4c700 x1877759534515200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772724.905332:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534515200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772724.905342:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772724.905347:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772724.905351:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534515200:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772724.905360:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772721/real 1790772724] req@ffff9c4efac4ed80 x1877759534515328/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772724.905382:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534515328/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772724.905394:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772724.905399:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772724.905404:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534515328:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772724.905412:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772721/real 1790772724] req@ffff9c4efac4dc00 x1877759534515456/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772724.905445:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534515456/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772724.905456:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772724.905461:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772724.905464:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534515456:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772724.905473:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772721/real 1790772724] req@ffff9c4efac4f800 x1877759534515584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772724.905486:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534515584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772724.905505:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772724.905510:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772724.905514:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534515584:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772724.905522:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772721/real 1790772724] req@ffff9c4efac4f480 x1877759534515712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772724.905537:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534515712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772724.905545:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772724.905550:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772724.905554:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534515712:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772724.905561:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772721/real 1790772724] req@ffff9c4efac4e680 x1877759534514944/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772724.905576:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534514944/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772724.905584:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772724.905588:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772724.905591:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534514944:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772724.905600:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772721/real 1790772724] req@ffff9c4efac4ea00 x1877759534515072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772724.905614:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534515072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772724.905632:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772724.905637:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772724.905641:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534515072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772724.905844:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772721/real 1790772724] req@ffff9c4efac4e300 x1877759534515840/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772724.905863:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534515840/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772724.905875:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772724.905879:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772724.905882:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534515840:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772724.968729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772724.968775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772724.968787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772724.968790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772725.032725:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772725.032727:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772725.032814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772725.160733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772725.160738:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772725.224704:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772725.288720:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772725.352782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772725.352798:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772725.352798:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772725.416704:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772725.480733:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772725.480733:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772725.480789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772725.608735:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772725.608740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772725.608740:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772725.608789:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772725.672722:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772725.672724:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772725.672731:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772725.672756:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772725.672871:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772725.672882:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772725.800756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772725.864721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772725.928780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772725.992724:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772725.992724:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772725.992759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772725.992816:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772726.056728:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772726.056762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772726.056938:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772726.184732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772726.184732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772726.248725:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772726.312820:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772726.376740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772726.376769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772726.376770:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772726.440709:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772726.504731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772726.504761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772726.504768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772726.632720:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772726.632762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772726.632767:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772726.632769:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772726.696708:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772726.696710:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772726.696769:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772726.696771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772726.696802:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772726.696816:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772726.824719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772726.888729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790772726.952727:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772726.952735:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772726.952738:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772726.952747:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1722 00000100:00080000:1.0:1790772726.952757:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772726.952767:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772726.952784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772726.952800:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534516096/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772726.952830:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772726.952834:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772726.952835:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772726.952839:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1722 00000100:00080000:1.0:1790772726.952841:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772726.952845:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772726.952853:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772726.952864:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534516224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772726.952876:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772726.952879:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772726.952880:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772726.952883:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1722 00000100:00080000:1.0:1790772726.952885:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772726.952888:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772726.952892:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772726.952900:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534516096:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772726.952905:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534516352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772726.952921:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772726.952924:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772726.952926:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772726.952929:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1722 00000100:00080000:1.0:1790772726.952932:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772726.952935:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772726.952938:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772726.952945:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534516480/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772726.952951:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772726.952953:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772726.952954:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772726.952957:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1722 00000100:00080000:1.0:1790772726.952960:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772726.952963:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772726.952967:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772726.952975:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534516608/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772726.952982:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772726.952986:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772726.952987:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772726.952990:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1722 00000100:00080000:1.0:1790772726.953001:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772726.953004:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772726.953008:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772726.953019:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534516736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772726.953025:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772726.953027:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772726.953034:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772726.953036:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772726.953039:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1722 00000100:00080000:1.0:1790772726.953044:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772726.953047:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772726.953050:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772726.953061:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534516864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772726.953079:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772726.953083:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772726.953086:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772726.953092:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1722 00000100:00080000:1.0:1790772726.953098:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772726.953104:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772726.953111:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772726/real 1790772726] req@ffff9c4efac4fb80 x1877759534516096/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772726.953114:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790772726.953150:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534516992/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772726.953157:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534516096/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772726.953191:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772726.953198:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772726.953199:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772726.953205:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772726.953212:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772726.953220:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1722 00000100:00100000:3.0:1790772726.953224:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534516096:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772726.953232:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772726.953245:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534516224:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772726.953246:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772726.953262:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772726.953284:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534517120/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772726.953303:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772726.953327:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772726/real 1790772726] req@ffff9c4efac4e300 x1877759534516224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772726.953343:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534516224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772726.953366:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772726.953371:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772726.953376:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534516224:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772726.953437:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534516352:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772726.953464:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772726.953479:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772726/real 1790772726] req@ffff9c4efac4ea00 x1877759534516352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772726.953491:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534516352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772726.953501:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772726.953505:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772726.953508:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534516352:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772726.953516:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534516480:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772726.953696:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772726.953713:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772726/real 1790772726] req@ffff9c4efac4e680 x1877759534516480/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772726.953730:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534516480/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772726.953742:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772726.953746:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772726.953750:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534516480:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772726.953759:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534516608:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772726.953787:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772726.953801:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772726/real 1790772726] req@ffff9c4efac4f480 x1877759534516608/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772726.953815:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534516608/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772726.953826:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772726.953831:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772726.953835:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534516608:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772726.953843:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534516736:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772726.953868:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772726.953882:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772726/real 1790772726] req@ffff9c4efac4f800 x1877759534516736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772726.953894:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534516736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772726.953916:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772726.953920:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772726.953924:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534516736:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772726.953931:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534516864:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772726.953954:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772726.953967:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772726/real 1790772726] req@ffff9c4efac4dc00 x1877759534516864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772726.953981:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534516864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772726.953991:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772726.953996:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772726.954000:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534516864:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772726.954008:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534516992:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772726.954032:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772726.954045:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772726/real 1790772726] req@ffff9c4efac4ed80 x1877759534516992/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772726.954059:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534516992/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772726.954071:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772726.954075:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772726.954079:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534516992:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772726.954086:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534517120:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772726.954109:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772726.954121:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772726/real 1790772726] req@ffff9c4efac4c700 x1877759534517120/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772726.954133:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534517120/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772726.954142:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772726.954144:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772726.954148:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534517120:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772727.016743:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772727.016744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772727.016744:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772727.016744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772727.080732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772727.080745:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772727.080840:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772727.208770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772727.208774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772727.272729:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772727.336777:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772727.400713:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772727.400717:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772727.400724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772727.464757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772727.528767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772727.528768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772727.528768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772727.656725:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772727.656730:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772727.656746:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772727.656753:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772727.720717:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772727.720755:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772727.720769:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772727.720783:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772727.721716:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772727.721728:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772727.848706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772727.912752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772727.977043:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772728.040727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772728.040756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772728.040773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772728.040775:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772728.104784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772728.104800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772728.104858:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772728.232794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772728.232825:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772728.296743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772728.360736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772728.424783:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772728.424790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772728.424790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772728.488765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772728.552738:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772728.552760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772728.552760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772728.680724:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772728.680728:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772728.680734:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772728.680789:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772728.744779:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772728.744802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772728.744820:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772728.744868:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772728.744929:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772728.744931:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772728.872706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772728.936717:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772729.000731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772729.064753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772729.064756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772729.064787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772729.064810:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772729.128779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772729.128787:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772729.128874:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772729.256741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772729.256753:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772729.320720:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772729.384763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772729.448756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772729.448757:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772729.448757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772729.512703:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772729.576715:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772729.576719:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772729.576722:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772729.704735:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772729.704757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772729.704758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772729.704785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772729.768749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772729.768755:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772729.768784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772729.768787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772729.768809:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772729.768821:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772729.896718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772729.960707:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772730.024702:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772730.088736:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772730.088737:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772730.088750:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772730.088763:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772730.152720:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772730.152749:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772730.152793:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772730.280752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772730.280786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772730.344713:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772730.408741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772730.472719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772730.472719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772730.472746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772730.536724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772730.600748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772730.600748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772730.600753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772730.728729:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772730.728737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772730.728738:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772730.728790:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772730.792740:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772730.792740:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772730.792768:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772730.792777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772730.792799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772730.792817:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772730.920696:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772730.984696:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772731.048706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772731.112741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772731.112765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772731.112766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772731.112771:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772731.176739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772731.176780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772731.176788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772731.304714:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772731.304773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772731.368753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772731.432716:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772731.496715:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772731.496721:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772731.496721:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772731.560706:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772731.624717:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772731.624723:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772731.624723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772731.752710:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772731.752730:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772731.752784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772731.752785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772731.816746:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772731.816747:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772731.816767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772731.816767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772731.816793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772731.816812:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772731.944710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772732.008719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772732.072749:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772732.072759:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1790772732.072762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772732.072765:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772732.072781:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1727 00000100:00080000:0.0:1790772732.072794:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772732.072809:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772732.072855:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534517248/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772732.072889:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772732.072893:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772732.072895:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772732.072900:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1727 00000100:00080000:0.0:1790772732.072903:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772732.072908:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772732.072920:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772732.072934:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534517376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772732.072945:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772732.072950:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772732.072952:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772732.072958:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1727 00000100:00080000:0.0:1790772732.072962:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772732.072968:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772732.072974:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772732.072982:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534517248:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772732.072989:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534517504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772732.073002:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772732.073006:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772732.073008:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772732.073013:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1727 00000100:00080000:0.0:1790772732.073017:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772732.073022:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772732.073028:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772732.073040:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534517632/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772732.073050:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772732.073055:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772732.073057:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772732.073062:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1727 00000100:00080000:0.0:1790772732.073067:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772732.073070:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772732.073075:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772732.073081:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772732.073095:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534517760/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772732.073106:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772732.073110:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772732.073113:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772732.073118:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1727 00000100:00100000:2.0:1790772732.073122:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772732/real 1790772732] req@ffff9c4ec7a7df80 x1877759534517248/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772732.073124:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772732.073135:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772732.073142:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772732.073145:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534517248/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772732.073159:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534517888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772732.073169:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772732.073171:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772732.073175:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772732.073176:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772732.073178:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772732.073183:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534517248:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772732.073184:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1727 00000100:00080000:0.0:1790772732.073190:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772732.073193:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534517376:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772732.073197:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772732.073202:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772732.073212:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534518016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772732.073220:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772732.073221:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772732.073226:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772732.073229:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772732.073234:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1727 00000100:00100000:2.0:1790772732.073237:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772732/real 1790772732] req@ffff9c4ec7a7f480 x1877759534517376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772732.073238:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772732.073246:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772732.073254:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772732.073262:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534517376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772732.073265:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534518144/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772732.073302:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772732.073305:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772732.073307:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772732.073312:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1727 00000100:00080000:2.0:1790772732.073313:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772732.073316:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772732.073319:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772732.073323:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772732.073325:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534517376:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790772732.073331:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772732.073342:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534518272/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772732.073360:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534517504:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772732.073378:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772732.073386:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772732/real 1790772732] req@ffff9c4ec7a7c380 x1877759534517504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772732.073395:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534517504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772732.073401:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772732.073404:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772732.073406:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534517504:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772732.073412:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534517632:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772732.073423:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772732.073430:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772732/real 1790772732] req@ffff9c4ec7a7d880 x1877759534517632/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772732.073438:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534517632/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772732.073445:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772732.073447:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772732.073449:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534517632:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772732.073453:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534517760:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772732.073463:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772732.073469:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772732/real 1790772732] req@ffff9c4ec7a7ed80 x1877759534517760/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772732.073477:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534517760/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772732.073484:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772732.073486:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772732.073488:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534517760:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772732.073492:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534517888:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772732.073502:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772732.073509:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772732/real 1790772732] req@ffff9c4ec7a7ca80 x1877759534517888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772732.073524:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534517888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772732.073537:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772732.073540:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772732.073542:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534517888:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772732.073546:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534518016:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772732.073558:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772732.073565:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772732/real 1790772732] req@ffff9c4ec7a7e300 x1877759534518016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772732.073574:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534518016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772732.073580:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772732.073582:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772732.073584:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534518016:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772732.073588:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534518144:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772732.073599:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772732.073605:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772732/real 1790772732] req@ffff9c4ec7a7fb80 x1877759534518144/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772732.073613:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534518144/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772732.073626:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772732.073628:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772732.073630:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534518144:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772732.073634:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534518272:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772732.073644:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772732.073669:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772732/real 1790772732] req@ffff9c4ec7a7c700 x1877759534518272/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772732.073677:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534518272/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772732.073685:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772732.073688:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772732.073690:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534518272:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772732.136716:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772732.136716:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772732.136720:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772732.136720:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772732.200718:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772732.200721:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772732.200778:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772732.328697:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772732.328708:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772732.392702:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772732.456710:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772732.520693:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772732.520713:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772732.520723:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772732.584759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772732.648712:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772732.648722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772732.648724:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772732.776708:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772732.776709:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772732.776721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772732.776723:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772732.840721:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772732.840722:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772732.840736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772732.840736:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772732.840761:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772732.840782:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772732.968708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772733.032699:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772733.096716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772733.160709:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772733.160726:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772733.160727:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772733.160728:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772733.224717:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772733.224760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772733.224780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772733.352712:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772733.352720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772733.416705:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772733.480793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772733.544702:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772733.544714:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772733.544716:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772733.608719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772733.672709:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772733.672715:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772733.672718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772733.800752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772733.800752:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772733.800752:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772733.800753:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772733.864713:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772733.864719:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772733.864725:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772733.864728:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772733.864753:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772733.864756:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772733.992715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772734.056775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772734.120702:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772734.184733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772734.184733:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772734.184734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772734.184738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772734.248725:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772734.248754:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772734.248823:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772734.376722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772734.376724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772734.440709:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772734.504709:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772734.568719:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772734.568724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772734.568725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772734.632709:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772734.696726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772734.696731:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772734.696781:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772734.824717:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772734.824721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772734.824735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772734.824735:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772734.888721:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772734.888721:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772734.888728:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772734.888732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772734.888759:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772734.888761:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772735.016725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772735.080733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772735.144706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772735.208719:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772735.208736:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772735.208736:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772735.208737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772735.272716:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772735.272758:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772735.272770:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772735.400715:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772735.400727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772735.464745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772735.528715:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772735.592749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772735.592749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772735.592749:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772735.656754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772735.720754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772735.720755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772735.720770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772735.848779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772735.848779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772735.848779:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772735.848786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772735.912733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772735.912738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772735.912746:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772735.912747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772735.912770:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772735.912779:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772736.040720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772736.104738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772736.168704:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772736.232751:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772736.232751:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772736.232752:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772736.232752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772736.296745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772736.296764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772736.296803:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772736.424730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772736.424736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772736.488748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772736.552700:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772736.616757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772736.616760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772736.616760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772736.680710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772736.744727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772736.744730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772736.744766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772736.872708:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772736.872725:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772736.872731:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772736.872748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772736.936742:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772736.936748:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772736.936753:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772736.936753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772736.936779:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772736.936788:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772737.064731:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772737.128742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772737.192734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772737.192740:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772737.192752:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772737.192758:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772737.192772:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1733 00000100:00080000:3.0:1790772737.192787:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772737.192804:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772737.192864:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534518400/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772737.192892:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772737.192897:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772737.192899:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772737.192905:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1733 00000100:00080000:3.0:1790772737.192910:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772737.192927:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772737.192941:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772737.192952:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534518400:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772737.192962:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534518528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772737.192974:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772737.192978:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772737.192980:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772737.192985:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1733 00000100:00080000:3.0:1790772737.192989:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772737.192995:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772737.193012:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772737.193025:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534518656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772737.193035:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772737.193040:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772737.193042:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772737.193046:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1733 00000100:00080000:3.0:1790772737.193051:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772737.193057:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772737.193060:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772737.193068:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772737.193083:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534518784/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772737.193094:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772737.193097:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772737.193098:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772737.193102:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1733 00000100:00080000:3.0:1790772737.193106:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772737.193111:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772737.193116:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772737.193119:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772737/real 1790772737] req@ffff9c4ef73c0e00 x1877759534518400/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772737.193129:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534518912/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772737.193137:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772737.193142:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534518400/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772737.193142:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772737.193147:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772737.193155:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1733 00000100:00080000:3.0:1790772737.193160:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772737.193165:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772737.193166:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790772737.193169:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772737.193201:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772737.193209:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534518400:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772737.193213:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534519040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772737.193226:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772737.193231:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772737.193233:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772737.193240:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1733 00000100:00080000:3.0:1790772737.193244:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772737.193251:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772737.193255:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534518528:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772737.193259:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772737.193274:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534519168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772737.193283:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772737.193288:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772737.193289:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772737.193293:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772737.193299:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1733 00000100:00080000:3.0:1790772737.193303:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772737.193307:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772737/real 1790772737] req@ffff9c4ef73c2d80 x1877759534518528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772737.193310:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772737.193317:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772737.193330:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534518528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772737.193333:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534519296/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772737.193361:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772737.193364:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772737.193365:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772737.193371:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772737.193371:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1733 00000100:00080000:3.0:1790772737.193374:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772737.193378:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772737.193381:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772737.193386:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534518528:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772737.193388:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772737.193398:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534518656:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772737.193403:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534519424/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772737.193418:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772737.193430:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772737/real 1790772737] req@ffff9c4ef73c1500 x1877759534518656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772737.193440:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534518656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772737.193446:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772737.193449:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772737.193452:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534518656:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772737.193456:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534518784:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772737.193468:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772737.193474:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772737/real 1790772737] req@ffff9c4ef73c0a80 x1877759534518784/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772737.193498:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534518784/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772737.193506:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772737.193508:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772737.193510:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534518784:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772737.193514:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534518912:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772737.193525:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772737.193531:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772737/real 1790772737] req@ffff9c4ef73c3800 x1877759534518912/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772737.193539:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534518912/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772737.193547:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772737.193549:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772737.193552:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534518912:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772737.193555:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534519040:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772737.193567:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772737.193573:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772737/real 1790772737] req@ffff9c4ef73c2300 x1877759534519040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772737.193580:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534519040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772737.193598:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772737.193601:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772737.193604:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534519040:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772737.193621:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534519168:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772737.193632:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772737.193639:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772737/real 1790772737] req@ffff9c4ef73c0700 x1877759534519168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772737.193648:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534519168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772737.193676:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772737.193678:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772737.193681:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534519168:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772737.193686:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534519296:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772737.193706:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772737.193718:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772737/real 1790772737] req@ffff9c4ef73c2a00 x1877759534519296/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772737.193733:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534519296/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772737.193746:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772737.193749:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772737.193753:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534519296:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772737.193758:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534519424:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772737.193774:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772737.193783:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772737/real 1790772737] req@ffff9c4ef73c1c00 x1877759534519424/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772737.193791:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534519424/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772737.193798:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772737.193801:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772737.193803:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534519424:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772737.256752:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772737.256752:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772737.256753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772737.256753:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772737.320734:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772737.320765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772737.320791:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772737.448718:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772737.448727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772737.512730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772737.576708:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772737.640718:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772737.640721:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772737.640770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772737.704727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772737.768754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772737.768754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772737.768769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772737.896762:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772737.896763:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772737.896763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772737.896785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772737.960723:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772737.960736:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772737.960744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772737.960744:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772737.960764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772737.960789:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772738.088767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772738.152735:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772738.216708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772738.280721:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772738.280722:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772738.280732:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772738.280735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772738.344728:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772738.344747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772738.344772:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772738.472723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772738.472724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772738.536711:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772738.600703:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772738.664719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772738.664728:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772738.664730:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772738.728710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772738.792705:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772738.792717:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772738.792718:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772738.920706:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772738.920711:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772738.920716:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772738.920729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772738.984703:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772738.984720:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772738.984729:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772738.984731:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772738.984749:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772738.984759:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772739.112710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772739.176702:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772739.240725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772739.304718:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772739.304719:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772739.304719:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772739.304733:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772739.368728:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772739.368760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772739.368797:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772739.496728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772739.496733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772739.560718:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772739.624702:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772739.688717:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772739.688730:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772739.688732:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772739.752712:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772739.816705:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772739.816727:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772739.816729:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772739.944729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772739.944729:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772739.944729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772739.944730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772740.008719:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772740.008719:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772740.008728:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772740.008732:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772740.008756:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772740.008764:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772740.136705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772740.200728:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772740.264728:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772740.328739:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772740.328739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772740.328750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772740.328750:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772740.392747:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772740.392751:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772740.392849:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772740.520725:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772740.520725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772740.584733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772740.648806:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772740.712701:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772740.712712:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772740.712736:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772740.776732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772740.840709:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772740.840722:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772740.840727:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772740.968714:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772740.968725:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772740.968726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772740.968731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772741.032722:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772741.032723:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772741.032723:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772741.032731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772741.032761:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772741.032779:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772741.160720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772741.224700:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772741.288724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772741.352708:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772741.352713:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772741.352716:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772741.352726:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772741.416706:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772741.416724:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772741.416832:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772741.544703:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772741.544717:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772741.608705:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772741.672697:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772741.736727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772741.736729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772741.736732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772741.800726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772741.864717:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772741.864722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772741.864740:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772741.992714:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772741.992720:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772741.992724:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772741.992731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772742.056732:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772742.056732:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772742.056733:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772742.056733:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772742.056765:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772742.056765:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772742.184720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772742.248720:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772742.312711:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772742.312721:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772742.312726:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772742.312738:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1738 00000100:00080000:3.0:1790772742.312749:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772742.312761:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772742.312803:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534519552/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772742.312828:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772742.312831:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772742.312834:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772742.312837:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1738 00000100:00080000:3.0:1790772742.312841:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772742.312845:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772742.312857:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772742.312868:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534519552:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772742.312873:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534519680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772742.312886:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772742.312890:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772742.312893:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772742.312898:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1738 00000100:00080000:3.0:1790772742.312903:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772742.312910:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772742.312916:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772742.312933:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534519808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772742.312945:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772742.312946:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772742.312951:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772742.312952:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772742.312958:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1738 00000100:00080000:3.0:1790772742.312962:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772742.312968:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772742.312974:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772742.312986:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534519936/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772742.312993:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772742/real 1790772742] req@ffff9c4ef73c0000 x1877759534519552/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772742.313000:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772742.313007:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772742.313009:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772742.313014:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1738 00000100:00080000:3.0:1790772742.313017:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772742.313019:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534519552/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772742.313020:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772742.313024:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772742.313033:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:2.0:1790772742.313038:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772742.313045:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772742.313046:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534520064/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772742.313052:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534519552:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772742.313064:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772742.313069:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772742.313071:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772742.313078:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1738 00000100:00100000:2.0:1790772742.313093:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534519680:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772742.313096:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772742.313107:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772742.313114:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790772742.313123:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772742.313132:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534520192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772742.313142:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772742.313145:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772742/real 1790772742] req@ffff9c4ef73c1c00 x1877759534519680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772742.313146:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772742.313150:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772742.313159:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1738 00000100:00080000:3.0:1790772742.313166:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772742.313171:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772742.313175:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534519680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772742.313177:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772742.313190:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534520320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772742.313198:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772742.313201:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772742.313203:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772742.313203:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772742.313212:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1738 00000100:00080000:2.0:1790772742.313215:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772742.313216:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772742.313225:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772742.313227:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534519680:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772742.313230:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772742.313242:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534519808:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772742.313246:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534520448/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772742.313255:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772742.313258:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772742.313260:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772742.313264:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1738 00000100:00080000:3.0:1790772742.313268:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772742.313270:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772742.313282:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772742/real 1790772742] req@ffff9c4ef73c2a00 x1877759534519808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772742.313286:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772742.313294:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772742.313301:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534519808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772742.313308:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534520576/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772742.313312:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772742.313320:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772742.313326:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534519808:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772742.313333:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534519936:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772742.313348:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772742.313355:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772742/real 1790772742] req@ffff9c4ef73c0700 x1877759534519936/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772742.313364:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534519936/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772742.313372:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772742.313375:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772742.313378:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534519936:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772742.313382:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534520064:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772742.313393:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772742.313399:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772742/real 1790772742] req@ffff9c4ef73c2300 x1877759534520064/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772742.313407:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534520064/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772742.313415:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772742.313417:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772742.313419:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534520064:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772742.313434:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534520192:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772742.313447:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772742.313454:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772742/real 1790772742] req@ffff9c4ef73c3800 x1877759534520192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772742.313462:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534520192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772742.313475:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772742.313477:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772742.313480:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534520192:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772742.313485:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534520320:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772742.313495:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772742.313501:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772742/real 1790772742] req@ffff9c4ef73c0a80 x1877759534520320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772742.313509:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534520320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772742.313515:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772742.313517:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772742.313519:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534520320:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772742.313523:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534520448:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772742.313534:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772742.313540:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772742/real 1790772742] req@ffff9c4ef73c1500 x1877759534520448/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772742.313548:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534520448/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772742.313555:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772742.313557:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772742.313559:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534520448:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772742.313563:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534520576:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772742.313573:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772742.313579:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772742/real 1790772742] req@ffff9c4ef73c2d80 x1877759534520576/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772742.313588:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534520576/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772742.313597:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772742.313600:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772742.313604:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534520576:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772742.376721:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772742.376728:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772742.376735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772742.376742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772742.440823:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772742.440886:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772742.440892:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772742.568706:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772742.568714:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772742.632702:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772742.696727:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772742.760715:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772742.760719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772742.761685:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772742.824713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772742.888721:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772742.888726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772742.888753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772743.016711:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772743.016722:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772743.016729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772743.016729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772743.080712:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772743.080719:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772743.080723:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772743.080727:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772743.080747:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772743.080750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772743.208729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772743.272699:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772743.336717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772743.400709:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772743.400716:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772743.400718:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772743.400722:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772743.464729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772743.464730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772743.464788:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772743.592720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772743.592756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772743.656709:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772743.720738:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772743.784735:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772743.784737:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772743.784762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772743.848725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772743.912716:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772743.912716:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772743.912764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772744.040720:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772744.040724:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772744.040800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772744.040801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772744.104723:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772744.104732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772744.104744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772744.104746:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772744.104777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772744.104802:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772744.232725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772744.296725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772744.360748:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772744.424709:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772744.424754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772744.424759:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772744.424785:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772744.488765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772744.488768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772744.488903:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772744.616710:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772744.616795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772744.680708:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772744.744725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772744.808719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772744.808725:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772744.808766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772744.872716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772744.936742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772744.936754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772744.936769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772745.064746:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772745.064757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772745.064803:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772745.064803:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772745.128738:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772745.128739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772745.128741:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772745.128747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772745.128797:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772745.128800:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772745.256722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772745.320714:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772745.384706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772745.448721:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772745.448726:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772745.448727:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772745.448761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00010000:00010000:1.0:1790772745.512796:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772745.512807:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772745.512854:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772745.640734:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772745.640781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772745.704714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772745.768756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772745.832718:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772745.832718:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772745.832718:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772745.896747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772745.960803:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772745.960817:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772745.960817:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772746.088753:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772746.088764:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772746.088819:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772746.088819:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772746.152730:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772746.152798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772746.152805:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772746.152850:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772746.152851:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772746.152880:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772746.280755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772746.344753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772746.408760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772746.472773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772746.472811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772746.472814:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772746.472820:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772746.536795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772746.536811:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772746.536965:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772746.664780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772746.664787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772746.728754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772746.792757:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772746.856787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772746.856792:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772746.856792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772746.920780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772746.984727:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772746.984770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772746.984797:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772747.112726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772747.112775:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772747.112791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772747.112795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772747.176816:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772747.176822:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772747.176851:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772747.176852:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772747.176902:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772747.176925:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772747.304793:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772747.368750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772747.432779:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772747.432791:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772747.432802:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772747.432810:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772747.432827:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1743 00000100:00080000:0.0:1790772747.432845:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772747.432865:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772747.432932:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534520704/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772747.432977:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772747.433008:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772747.433012:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772747.433021:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1743 00000100:00080000:0.0:1790772747.433029:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772747.433038:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772747.433055:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772747.433080:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534520832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772747.433089:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534520704:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772747.433107:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772747.433114:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772747.433117:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772747.433125:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1743 00000100:00080000:0.0:1790772747.433130:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772747.433138:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772747.433145:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772747.433160:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534520960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772747.433174:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772747.433179:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772747.433182:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772747.433188:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1743 00000100:00080000:0.0:1790772747.433198:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772747.433205:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772747.433212:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772747.433227:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534521088/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772747.433244:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772747.433245:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772747.433256:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772747.433259:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772747.433266:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1743 00000100:00080000:0.0:1790772747.433273:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772747.433281:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772747.433287:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772747.433304:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534521216/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772747.433317:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772747.433326:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772747/real 1790772747] req@ffff9c4ec7a7d180 x1877759534520704/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772747.433327:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772747.433333:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772747.433348:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1743 00000100:00080000:0.0:1790772747.433359:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772747.433369:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772747.433376:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772747.433379:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534520704/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772747.433395:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534521344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772747.433402:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772747.433418:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772747.433419:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772747.433432:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1790772747.433437:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534520704:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772747.433439:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772747.433457:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1743 00000100:00080000:0.0:1790772747.433463:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772747.433469:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772747.433475:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772747.433489:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534520832:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772747.433495:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534521472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772747.433512:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772747.433518:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772747.433521:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772747.433528:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1743 00000800:00000100:2.0:1790772747.433535:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772747.433536:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772747.433549:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772747.433558:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772747.433577:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772747/real 1790772747] req@ffff9c4ec7a7c700 x1877759534520832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772747.433583:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534521600/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772747.433639:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772747.433647:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534520832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772747.433709:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772747.433716:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772747.433721:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772747.433728:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1743 00000100:00080000:2.0:1790772747.433736:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790772747.433738:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772747.433749:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534520832:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772747.433756:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772747.433767:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534520960:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790772747.433768:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772747.433803:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534521728/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772747.433839:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772747.433877:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772747/real 1790772747] req@ffff9c4ec7a7fb80 x1877759534520960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772747.433907:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534520960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772747.433927:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772747.433936:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772747.433943:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534520960:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772747.433957:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534521088:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772747.434019:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772747.434050:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772747/real 1790772747] req@ffff9c4ec7a7e300 x1877759534521088/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772747.434067:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534521088/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772747.434081:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772747.434086:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772747.434091:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534521088:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772747.434100:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534521216:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772747.434124:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772747.434138:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772747/real 1790772747] req@ffff9c4ec7a7ca80 x1877759534521216/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772747.434154:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534521216/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772747.434166:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772747.434170:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772747.434175:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534521216:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772747.434183:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534521344:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772747.434214:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772747.434227:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772747/real 1790772747] req@ffff9c4ec7a7ed80 x1877759534521344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772747.434242:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534521344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772747.434263:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772747.434267:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772747.434271:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534521344:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772747.434299:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534521472:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772747.434321:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772747.434334:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772747/real 1790772747] req@ffff9c4ec7a7d880 x1877759534521472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772747.434349:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534521472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772747.434361:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772747.434366:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772747.434371:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534521472:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772747.434378:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534521600:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772747.434396:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772747.434408:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772747/real 1790772747] req@ffff9c4ec7a7c380 x1877759534521600/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772747.434422:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534521600/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772747.434436:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772747.434440:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772747.434445:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534521600:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772747.434453:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534521728:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772747.434472:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772747.434484:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772747/real 1790772747] req@ffff9c4ec7a7f480 x1877759534521728/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772747.434499:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534521728/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772747.434511:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772747.434515:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772747.434520:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534521728:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772747.496814:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772747.496818:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772747.496847:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772747.496855:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772747.560807:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772747.560857:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772747.560957:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772747.688764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772747.688827:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772747.752776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772747.816911:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772747.880779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772747.880876:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772747.880879:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772747.944776:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772748.008813:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772748.008828:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772748.008834:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772748.136798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772748.136997:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772748.137137:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772748.137307:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772748.200779:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772748.200809:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772748.200870:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772748.200906:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772748.200910:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772748.200991:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772748.328775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772748.392707:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772748.456746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772748.520730:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772748.520737:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772748.520737:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772748.520786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772748.584821:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772748.584906:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772748.584926:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772748.712731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772748.712850:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772748.776745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772748.840767:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772748.904833:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772748.904852:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772748.904854:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772748.968726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772749.032733:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772749.032733:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772749.032824:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772749.160740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772749.160740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772749.160775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772749.160829:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772749.224767:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772749.224787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772749.224830:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772749.224835:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772749.224880:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772749.224881:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772749.352772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772749.416735:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772749.480767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772749.544794:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772749.544843:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772749.544863:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772749.544868:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772749.608758:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772749.608814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772749.608849:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772749.736790:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772749.736838:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772749.800759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772749.864762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772749.928719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772749.928777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772749.928793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772749.992758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772750.056779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772750.056805:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772750.056827:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772750.184858:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772750.184874:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772750.185314:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772750.185330:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772750.248778:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772750.248804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772750.248845:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772750.248861:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772750.248865:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772750.248895:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772750.376775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772750.440768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772750.504795:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772750.568792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772750.568798:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772750.568800:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772750.568813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772750.632817:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772750.632869:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772750.632982:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772750.760770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772750.760771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772750.824756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772750.888778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772750.952775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772750.952784:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772750.952825:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772751.016751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772751.080752:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772751.080752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772751.080799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772751.208770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772751.208770:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772751.208770:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772751.208965:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772751.272725:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772751.272746:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772751.272756:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772751.272798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772751.272800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772751.272824:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772751.400769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772751.464744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772751.528767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772751.592785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772751.592817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772751.592818:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772751.592828:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772751.656782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772751.656823:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772751.656980:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772751.784751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772751.784767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772751.848755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772751.912781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772751.976728:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772751.976780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772751.976797:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772752.040778:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772752.104797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772752.104853:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772752.104853:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772752.232779:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772752.232870:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772752.232925:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772752.232941:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772752.296753:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772752.296772:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772752.296788:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772752.296804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772752.296834:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772752.296853:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772752.424826:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772752.488755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772752.552789:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772752.552804:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772752.552815:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772752.552822:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772752.552838:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1748 00000100:00080000:2.0:1790772752.552854:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772752.552873:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772752.552971:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534521856/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772752.553014:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772752.553020:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772752.553024:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772752.553030:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1748 00000100:00080000:2.0:1790772752.553036:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772752.553043:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772752.553059:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772752.553091:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534521984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772752.553109:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534521856:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772752.553112:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772752.553125:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772752.553128:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772752.553137:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1748 00000100:00080000:2.0:1790772752.553143:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772752.553151:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772752.553159:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772752.553179:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534522112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772752.553194:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772752.553211:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772752.553214:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772752.553220:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1748 00000100:00080000:2.0:1790772752.553226:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772752.553231:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772752.553239:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772752.553244:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772752.553270:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534522240/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772752.553285:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772752.553290:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772752.553293:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772752.553299:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1748 00000100:00080000:2.0:1790772752.553304:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772752.553312:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772752.553321:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772752/real 1790772752] req@ffff9c4efbcc1500 x1877759534521856/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790772752.553323:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772752.553349:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534522368/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772752.553375:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772752.553376:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534521856/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772752.553385:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772752.553391:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772752.553407:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1748 00000100:00080000:2.0:1790772752.553415:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772752.553416:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772752.553425:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772752.553433:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790772752.553436:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772752.553445:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534521856:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772752.553456:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534522496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772752.553471:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772752.553476:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772752.553480:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772752.553488:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534521984:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772752.553494:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1748 00000100:00080000:2.0:1790772752.553504:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772752.553513:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772752.553521:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1790772752.553539:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772752.553553:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534522624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772752.553563:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772752/real 1790772752] req@ffff9c4efbcc2a00 x1877759534521984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772752.553580:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772752.553590:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772752.553594:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772752.553602:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534521984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772752.553607:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1748 00000100:00080000:2.0:1790772752.553620:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772752.553634:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772752.553643:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:3.0:1790772752.553649:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1790772752.553699:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534522752/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772752.553721:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772752.553738:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534521984:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772752.553746:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772752.553756:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1790772752.553757:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534522112:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772752.553762:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772752.553774:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1748 00000100:00080000:2.0:1790772752.553779:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1790772752.553802:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772752.553803:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772752.553817:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772752.553831:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772752/real 1790772752] req@ffff9c4efbcc1c00 x1877759534522112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772752.553840:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534522880/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772752.553908:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534522112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772752.553929:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772752.553937:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772752.553945:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534522112:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772752.553954:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534522240:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772752.553994:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772752.554008:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772752/real 1790772752] req@ffff9c4efbcc1880 x1877759534522240/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772752.554024:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534522240/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772752.554037:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772752.554041:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772752.554045:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534522240:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772752.554053:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534522368:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772752.554073:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772752.554085:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772752/real 1790772752] req@ffff9c4efbcc2300 x1877759534522368/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772752.554100:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534522368/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772752.554112:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772752.554116:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772752.554120:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534522368:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772752.554127:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534522496:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772752.554147:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772752.554159:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772752/real 1790772752] req@ffff9c4efbcc2680 x1877759534522496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772752.554184:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534522496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772752.554216:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772752.554220:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772752.554224:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534522496:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772752.554250:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534522624:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772752.554274:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772752.554286:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772752/real 1790772752] req@ffff9c4efbcc0700 x1877759534522624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772752.554302:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534522624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772752.554313:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772752.554317:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772752.554321:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534522624:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772752.554328:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534522752:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772752.554348:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772752.554359:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772752/real 1790772752] req@ffff9c4efbcc1f80 x1877759534522752/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772752.554374:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534522752/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772752.554404:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772752.554409:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772752.554413:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534522752:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772752.554420:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534522880:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772752.554439:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772752.554451:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772752/real 1790772752] req@ffff9c4efbcc0a80 x1877759534522880/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772752.554466:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534522880/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772752.554478:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772752.554482:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772752.554486:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534522880:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772752.616776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772752.616776:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772752.616795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772752.616802:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772752.680785:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772752.680840:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772752.680941:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772752.808787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772752.808792:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772752.872772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772752.936763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772753.000763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772753.000787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772753.000800:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772753.064754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772753.128743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772753.128766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772753.128778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772753.256952:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772753.257051:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772753.257123:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772753.257271:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772753.320807:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772753.320807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772753.320842:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772753.320894:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772753.320945:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772753.320945:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772753.448781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772753.512782:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772753.576785:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772753.640786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772753.640786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772753.640801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772753.640803:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772753.704796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772753.704796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772753.704933:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772753.832767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772753.832811:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772753.896754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772753.960801:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772754.024777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772754.024807:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772754.024810:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772754.088807:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772754.152762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772754.152774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772754.152806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772754.280751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772754.280776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772754.280809:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772754.280810:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772754.344778:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772754.344827:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772754.344835:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772754.344872:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772754.344905:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772754.344909:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772754.472746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772754.536755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772754.600744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772754.664770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772754.664777:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772754.664791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772754.664795:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772754.728780:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772754.728813:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772754.729008:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772754.856752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772754.856762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772754.920790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772754.984896:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772755.048768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772755.048770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772755.048809:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772755.112768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772755.176764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772755.176774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772755.176775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772755.304735:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772755.304735:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772755.304735:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772755.304753:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772755.368725:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772755.368725:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772755.368729:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772755.368729:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772755.368758:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772755.368758:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772755.496711:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772755.560706:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772755.624701:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772755.688749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772755.688750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772755.688760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772755.688782:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772755.752799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772755.752919:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772755.752930:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772755.880787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772755.880798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772755.944787:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772756.008722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772756.072747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772756.072751:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772756.072761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772756.136786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772756.200774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772756.200780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772756.200843:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772756.328693:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772756.328775:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772756.328839:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772756.328843:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772756.392774:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772756.392793:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772756.392874:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772756.392897:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772756.392912:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772756.392913:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772756.520721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772756.584747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772756.648751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772756.712747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772756.712761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772756.712796:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772756.712798:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772756.776784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772756.776872:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772756.776894:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772756.904752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772756.904780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772756.968755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772757.032789:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772757.096779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772757.096861:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772757.096863:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772757.160762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772757.224762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772757.224839:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772757.224864:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772757.352775:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772757.352813:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772757.352854:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772757.352854:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772757.416764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772757.416827:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772757.416828:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772757.416890:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772757.416895:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772757.416928:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772757.544744:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772757.608779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772757.672813:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772757.672920:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772757.672931:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772757.672938:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772757.672954:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1753 00000100:00080000:3.0:1790772757.672972:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772757.672992:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772757.673052:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534523008/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772757.673099:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772757.673104:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772757.673107:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772757.673113:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1753 00000100:00080000:3.0:1790772757.673118:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772757.673125:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772757.673142:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772757.673161:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534523136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772757.673179:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772757.673182:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534523008:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772757.673186:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772757.673193:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772757.673208:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1753 00000100:00080000:3.0:1790772757.673213:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772757.673218:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772757.673225:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772757.673242:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534523264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772757.673257:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772757.673263:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772757.673266:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772757.673271:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1753 00000100:00080000:3.0:1790772757.673298:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772757.673306:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772757.673313:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:2.0:1790772757.673325:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772757.673333:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534523392/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772757.673348:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772757.673354:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772757.673358:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772757.673365:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1753 00000100:00080000:3.0:1790772757.673369:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772757.673375:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772757.673382:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772757.673399:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534523520/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772757.673412:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772757/real 1790772757] req@ffff9c4ef73c0e00 x1877759534523008/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772757.673448:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772757.673455:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772757.673459:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772757.673470:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1753 00000100:00100000:2.0:1790772757.673472:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534523008/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772757.673479:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772757.673498:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772757.673512:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772757.673517:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772757.673529:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772757.673533:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534523648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772757.673543:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534523008:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772757.673564:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772757.673570:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772757.673574:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772757.673583:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1753 00000100:00080000:3.0:1790772757.673587:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772757.673594:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772757.673602:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534523136:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772757.673610:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772757.673632:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534523776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772757.673648:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772757.673686:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772757.673717:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772757/real 1790772757] req@ffff9c4ef73c2d80 x1877759534523136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772757.673719:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772757.673726:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772757.673741:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1753 00000100:00080000:3.0:1790772757.673751:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772757.673764:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772757.673774:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772757.673776:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534523136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772757.673829:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534523904/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772757.673855:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772757.673875:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772757.673881:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772757.673886:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534523136:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772757.673892:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772757.673899:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772757.673906:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534523264:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772757.673923:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1753 00000100:00080000:3.0:1790772757.673933:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772757.673949:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772757.673958:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790772757.673966:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772757.673986:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772757/real 1790772757] req@ffff9c4ef73c1500 x1877759534523264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772757.673995:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534524032/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772757.674032:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534523264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772757.674048:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772757.674054:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772757.674060:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534523264:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772757.674069:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534523392:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772757.674095:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772757.674107:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772757/real 1790772757] req@ffff9c4ef73c0a80 x1877759534523392/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772757.674124:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534523392/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772757.674138:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772757.674142:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772757.674146:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534523392:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772757.674156:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534523520:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772757.674191:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772757.674214:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772757/real 1790772757] req@ffff9c4ef73c3800 x1877759534523520/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772757.674241:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534523520/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772757.674262:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772757.674269:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772757.674275:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534523520:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772757.674284:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534523648:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772757.674317:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772757.674332:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772757/real 1790772757] req@ffff9c4ef73c2300 x1877759534523648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772757.674348:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534523648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772757.674379:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772757.674384:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772757.674388:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534523648:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772757.674427:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534523776:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772757.674450:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772757.674462:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772757/real 1790772757] req@ffff9c4ef73c0700 x1877759534523776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772757.674489:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534523776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772757.674500:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772757.674504:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772757.674509:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534523776:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772757.674516:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534523904:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772757.674536:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772757.674547:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772757/real 1790772757] req@ffff9c4ef73c2a00 x1877759534523904/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772757.674563:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534523904/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772757.674576:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772757.674580:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772757.674586:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534523904:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772757.674593:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534524032:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772757.674613:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772757.674624:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772757/real 1790772757] req@ffff9c4ef73c1c00 x1877759534524032/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772757.674639:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534524032/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772757.674680:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772757.674685:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772757.674689:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534524032:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772757.736817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772757.736826:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772757.736826:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772757.736871:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772757.800847:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772757.800848:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772757.800975:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772757.928760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772757.928843:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772757.992740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772758.056806:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772758.120755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772758.120829:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772758.120838:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772758.184737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772758.248756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772758.248819:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772758.248838:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772758.376823:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772758.376823:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772758.376852:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772758.376906:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772758.440819:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772758.440876:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772758.440888:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772758.440892:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772758.440931:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772758.440971:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772758.568763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772758.632756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772758.696816:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772758.760795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772758.760842:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772758.760850:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772758.760855:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00010000:00010000:1.0:1790772758.824838:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772758.824845:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772758.824904:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772758.952804:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772758.952867:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772759.016738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772759.080830:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772759.144794:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772759.144819:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772759.144836:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772759.208789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772759.272730:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772759.272817:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772759.272828:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772759.400793:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772759.400803:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772759.400830:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772759.400870:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772759.464800:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772759.464814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772759.464826:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772759.464840:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772759.464871:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772759.464889:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772759.592788:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772759.656739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772759.720788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772759.784780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772759.784812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772759.784829:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772759.784867:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772759.848835:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772759.848906:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772759.848910:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772759.976787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772759.976808:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772760.040792:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772760.104779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772760.168761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772760.168779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772760.168779:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772760.232793:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772760.296806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772760.296836:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772760.296837:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772760.424806:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772760.424836:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772760.424851:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772760.424872:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772760.488787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772760.488813:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772760.488858:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772760.488871:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772760.488872:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772760.488911:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772760.616721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772760.680778:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772760.744808:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772760.808813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772760.808832:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772760.808839:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772760.808851:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772760.872848:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772760.872876:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772760.872880:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772761.000842:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772761.000847:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772761.064828:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772761.128822:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772761.192762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772761.192832:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772761.192845:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772761.256822:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772761.320822:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772761.320822:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772761.320833:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772761.448826:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772761.448835:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772761.448836:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772761.448848:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772761.512786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772761.512823:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772761.512851:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772761.512871:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772761.512894:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772761.512906:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772761.640770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772761.704785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772761.768785:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772761.832775:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772761.832838:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772761.832857:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772761.832907:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772761.896796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772761.896839:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772761.897005:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772762.024800:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772762.024815:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772762.088786:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772762.152901:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772762.216762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772762.216841:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772762.216844:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772762.280823:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772762.344804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772762.344827:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772762.344834:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772762.472785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772762.472794:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772762.472801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772762.472807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772762.536818:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772762.536835:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772762.536884:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772762.536897:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772762.536906:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772762.536914:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772762.664756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772762.728795:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772762.792792:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772762.792806:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772762.792817:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772762.792824:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772762.792842:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1758 00000100:00080000:3.0:1790772762.792860:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772762.792892:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772762.792961:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534524160/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772762.793029:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772762.793037:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772762.793041:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772762.793052:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1758 00000100:00080000:3.0:1790772762.793061:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772762.793076:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772762.793097:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772762.793101:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534524160:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772762.793137:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534524288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772762.793167:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772762.793176:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772762.793181:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772762.793192:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1758 00000100:00080000:3.0:1790772762.793203:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772762.793215:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772762.793225:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790772762.793227:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772762.793269:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534524416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772762.793286:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772762.793292:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772762.793295:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772762.793303:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1758 00000100:00080000:3.0:1790772762.793312:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772762.793325:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772762.793327:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772762/real 1790772762] req@ffff9c4ef73c0000 x1877759534524160/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772762.793337:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772762.793367:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534524544/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772762.793374:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534524160/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772762.793411:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772762.793418:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772762.793419:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772762.793423:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772762.793430:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772762.793435:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1758 00000100:00100000:2.0:1790772762.793445:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534524160:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772762.793446:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772762.793463:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772762.793470:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772762.793485:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534524672/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772762.793498:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534524288:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772762.793503:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772762.793510:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772762.793513:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772762.793520:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1758 00000100:00080000:3.0:1790772762.793526:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772762.793549:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772762.793550:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772762.793564:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772762.793581:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534524800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772762.793598:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772762/real 1790772762] req@ffff9c4ef73c1c00 x1877759534524288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772762.793611:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772762.793627:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772762.793634:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772762.793641:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534524288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772762.793643:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1758 00000100:00080000:3.0:1790772762.793725:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772762.793745:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772762.793751:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790772762.793758:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772762.793766:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772762.793779:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534524288:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772762.793791:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534524928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772762.793797:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534524416:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772762.793824:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772762.793830:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772762.793835:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772762.793844:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1758 00000100:00080000:3.0:1790772762.793852:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772762.793856:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772762.793865:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772762.793876:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772762.793883:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772762/real 1790772762] req@ffff9c4ef73c2a00 x1877759534524416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772762.793901:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534525056/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772762.793911:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534524416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772762.793938:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772762.793945:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772762.793946:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772762.793955:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772762.793960:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772762.793967:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1758 00000100:00100000:2.0:1790772762.793970:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534524416:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772762.793979:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772762.793987:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534524544:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772762.793992:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772762.794005:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772762.794027:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534525184/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772762.794036:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772762.794059:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772762/real 1790772762] req@ffff9c4ef73c0700 x1877759534524544/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772762.794080:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534524544/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772762.794096:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772762.794101:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772762.794105:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534524544:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772762.794131:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534524672:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772762.794155:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772762.794168:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772762/real 1790772762] req@ffff9c4ef73c2300 x1877759534524672/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772762.794183:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534524672/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772762.794200:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772762.794207:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772762.794213:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534524672:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772762.794225:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534524800:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772762.794265:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772762.794290:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772762/real 1790772762] req@ffff9c4ef73c3800 x1877759534524800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772762.794316:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534524800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772762.794341:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772762.794346:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772762.794352:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534524800:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772762.794361:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534524928:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772762.794391:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772762.794406:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772762/real 1790772762] req@ffff9c4ef73c0a80 x1877759534524928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772762.794422:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534524928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772762.794433:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772762.794437:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772762.794441:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534524928:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772762.794448:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534525056:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772762.794469:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772762.794481:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772762/real 1790772762] req@ffff9c4ef73c1500 x1877759534525056/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772762.794497:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534525056/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772762.794511:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772762.794515:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772762.794519:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534525056:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772762.794527:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534525184:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772762.794546:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772762.794559:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772762/real 1790772762] req@ffff9c4ef73c2d80 x1877759534525184/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772762.794575:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534525184/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772762.794587:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772762.794591:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772762.794595:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534525184:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772762.856777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772762.856802:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772762.856802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772762.856818:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772762.920780:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772762.920828:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772762.920920:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772763.048788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772763.048788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772763.112829:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772763.176839:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772763.240779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772763.240781:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772763.240807:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772763.304827:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772763.368781:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772763.368787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772763.368822:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772763.496769:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772763.496794:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772763.496805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772763.496807:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772763.560811:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772763.560811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772763.560817:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772763.560842:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772763.560892:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772763.560896:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772763.688793:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772763.752787:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772763.816765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772763.880813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772763.880827:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772763.880829:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772763.880837:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772763.944816:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772763.944850:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772763.944913:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772764.072815:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772764.072833:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772764.136793:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772764.200772:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772764.264809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772764.264809:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772764.264814:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772764.328799:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772764.392785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772764.392812:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772764.392816:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772764.520791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772764.520800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772764.520829:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772764.520835:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772764.584758:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772764.584792:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772764.584824:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772764.584828:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772764.584832:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772764.584867:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772764.712774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772764.776744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772764.840767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772764.904776:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772764.904776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772764.904784:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772764.904804:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772764.968769:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772764.968815:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772764.968984:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772765.096785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772765.096785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772765.160765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772765.224785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772765.288747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772765.288784:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772765.288796:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772765.352773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772765.416769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772765.416809:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772765.416817:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772765.544747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772765.544795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772765.544804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772765.544815:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772765.608788:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772765.608796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772765.608831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772765.608838:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772765.608873:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772765.608880:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772765.736777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772765.800750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772765.864741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772765.928735:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772765.928736:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772765.928736:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772765.928738:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772765.992736:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772765.992736:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772765.992877:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772766.120711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772766.120731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772766.184726:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772766.248724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772766.312712:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772766.312719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772766.312720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772766.376702:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772766.440703:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772766.440709:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772766.440713:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772766.568698:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772766.568714:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772766.568714:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772766.568725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772766.632698:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772766.632714:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772766.632716:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772766.632732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772766.632744:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772766.632750:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772766.760741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772766.824696:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772766.888694:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772766.952731:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772766.952734:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772766.952736:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772766.952739:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772767.016712:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772767.016724:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772767.016904:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772767.144704:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772767.144721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772767.208721:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772767.272709:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772767.336709:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772767.336718:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772767.336719:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772767.400709:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772767.464736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772767.464736:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772767.464736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772767.592761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772767.592772:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772767.592776:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772767.592781:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772767.656779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772767.656803:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772767.656815:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772767.656818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772767.656850:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772767.656874:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772767.784722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772767.848755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772767.912733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772767.912841:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772767.912847:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772767.912852:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772767.912864:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1763 00000100:00080000:2.0:1790772767.912877:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772767.912891:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772767.912942:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534525312/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772767.912971:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772767.912974:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772767.912976:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772767.912979:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1763 00000100:00080000:2.0:1790772767.912982:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772767.912987:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772767.912999:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772767.913020:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534525440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772767.913022:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534525312:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772767.913040:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772767.913045:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772767.913047:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772767.913053:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1763 00000100:00080000:2.0:1790772767.913058:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772767.913065:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772767.913092:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772767.913123:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534525568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772767.913145:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1790772767.913148:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772767.913159:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772767.913164:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772767.913174:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1763 00000100:00080000:2.0:1790772767.913183:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772767.913192:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772767.913201:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772767.913220:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534525696/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772767.913234:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772767.913240:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772767.913248:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772767.913267:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772767/real 1790772767] req@ffff9c4efbcc3100 x1877759534525312/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772767.913274:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1763 00000100:00080000:2.0:1790772767.913282:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772767.913293:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772767.913305:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772767.913347:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534525824/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772767.913357:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534525312/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772767.913370:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772767.913385:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772767.913391:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772767.913402:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1763 00000100:00080000:3.0:1790772767.913404:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772767.913408:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772767.913416:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772767.913423:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772767.913432:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534525312:192.168.204.150@tcp:250:kworker.0 00000008:00100000:2.0:1790772767.913436:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772767.913459:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534525952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772767.913472:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772767.913478:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1790772767.913485:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534525440:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772767.913486:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772767.913504:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1763 00000100:00080000:2.0:1790772767.913514:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772767.913520:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772767.913529:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772767.913547:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534526080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772767.913561:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772767.913580:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772767.913630:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772767.913632:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772767.913638:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772767/real 1790772767] req@ffff9c4efbcc0380 x1877759534525440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772767.913639:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1763 00000100:00080000:2.0:1790772767.913645:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772767.913694:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772767.913702:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772767.913721:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534525440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772767.913728:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534526208/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772767.913771:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772767.913776:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772767.913780:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772767.913788:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1763 00000100:00080000:3.0:1790772767.913791:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772767.913794:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772767.913830:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772767.913831:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772767.913838:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772767.913841:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534525440:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772767.913849:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534526336/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772767.913854:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534525568:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772767.913883:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772767.913893:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772767/real 1790772767] req@ffff9c4efbcc0a80 x1877759534525568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772767.913902:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534525568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772767.913909:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772767.913912:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772767.913915:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534525568:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772767.913920:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534525696:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772767.913933:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772767.913939:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772767/real 1790772767] req@ffff9c4efbcc1f80 x1877759534525696/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772767.913953:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534525696/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772767.913961:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772767.913963:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772767.913965:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534525696:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772767.913969:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534525824:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772767.913980:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772767.913987:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772767/real 1790772767] req@ffff9c4efbcc0700 x1877759534525824/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772767.913995:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534525824/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772767.914002:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772767.914004:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772767.914007:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534525824:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772767.914011:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534525952:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772767.914022:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772767.914028:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772767/real 1790772767] req@ffff9c4efbcc2680 x1877759534525952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772767.914036:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534525952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772767.914054:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772767.914057:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772767.914060:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534525952:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772767.914101:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534526080:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772767.914126:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772767.914139:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772767/real 1790772767] req@ffff9c4efbcc2300 x1877759534526080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772767.914155:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534526080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772767.914166:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772767.914170:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772767.914174:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534526080:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772767.914182:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534526208:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772767.914203:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772767.914222:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772767/real 1790772767] req@ffff9c4efbcc1880 x1877759534526208/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772767.914262:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534526208/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772767.914283:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772767.914290:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772767.914299:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534526208:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772767.914333:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534526336:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772767.914371:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772767.914388:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772767/real 1790772767] req@ffff9c4efbcc1c00 x1877759534526336/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772767.914404:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534526336/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772767.914418:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772767.914422:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772767.914427:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534526336:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772767.976747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772767.976762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772767.976777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772767.976790:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772768.040763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772768.040786:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772768.040928:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772768.168790:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772768.168801:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772768.232776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772768.296809:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772768.360720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772768.360725:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772768.360732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772768.424710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772768.488703:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772768.488717:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772768.488792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772768.616854:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772768.616854:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772768.616858:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772768.616887:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772768.680802:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772768.680813:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772768.680829:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772768.680831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772768.680872:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772768.680878:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772768.808791:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772768.872771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772768.936747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772769.000786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772769.000829:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772769.000850:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772769.000852:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772769.064791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772769.064805:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772769.065068:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772769.192797:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772769.192797:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772769.256756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772769.320933:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772769.384720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772769.384760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772769.384820:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772769.448704:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772769.512746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772769.512759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772769.512813:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772769.640732:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772769.640773:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772769.640774:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772769.640776:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772769.704744:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772769.704807:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772769.704836:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772769.704868:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772769.704881:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772769.704895:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772769.832737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772769.896720:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772769.960712:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772770.024754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772770.024762:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772770.024781:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772770.024797:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772770.088752:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772770.088859:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772770.088923:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772770.216769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772770.216779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772770.280749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772770.344752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772770.408720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772770.408725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772770.408726:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772770.472728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772770.536753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772770.536757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772770.536798:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772770.664762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772770.664783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772770.664789:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772770.664789:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772770.728781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772770.728793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772770.728803:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772770.728829:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772770.728840:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772770.728878:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772770.856781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772770.920784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772770.984787:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772771.048732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772771.048745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772771.048761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772771.048765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772771.112814:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772771.112852:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772771.112923:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772771.240742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772771.240749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772771.304817:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772771.368752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772771.432753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772771.432755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772771.432763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772771.496739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772771.560721:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772771.560727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772771.560729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772771.688720:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772771.688726:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772771.688732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772771.688740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772771.752722:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772771.752723:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772771.752728:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772771.752729:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772771.752756:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772771.752762:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772771.880735:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772771.944764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772772.008748:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772772.072777:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772772.072777:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772772.072778:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772772.072821:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772772.136709:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772772.136740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772772.136806:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772772.264745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772772.264772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772772.328728:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772772.392817:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772772.456774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772772.456791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772772.456791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772772.520757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772772.584778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772772.584798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772772.584798:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772772.712784:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772772.712794:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772772.712827:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772772.712830:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772772.776835:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772772.776838:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772772.776913:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772772.776915:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772772.776947:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772772.776948:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772772.904748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772772.968740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772773.032787:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772773.032795:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772773.032805:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772773.032812:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772773.032827:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1768 00000100:00080000:1.0:1790772773.032844:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772773.032861:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772773.032924:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534526464/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772773.032968:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772773.032975:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772773.032978:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772773.032985:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1768 00000100:00080000:1.0:1790772773.032992:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772773.033002:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772773.033020:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772773.033032:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534526464:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772773.033047:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534526592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772773.033061:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772773.033065:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772773.033068:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772773.033073:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1768 00000100:00080000:1.0:1790772773.033078:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772773.033083:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772773.033090:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772773.033104:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534526720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772773.033121:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772773.033126:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772773.033128:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772773.033135:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1768 00000100:00080000:1.0:1790772773.033140:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772773.033147:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772773.033150:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772773.033154:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772773.033167:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534526848/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772773.033177:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772773.033182:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772773.033184:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772773.033190:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1768 00000100:00080000:1.0:1790772773.033194:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772773.033198:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772773.033202:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772773.033214:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534526976/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772773.033224:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772773.033230:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1790772773.033230:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772773/real 1790772773] req@ffff9c4efac4df80 x1877759534526464/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772773.033236:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772773.033248:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1768 00000100:00080000:1.0:1790772773.033255:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772773.033262:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772773.033268:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772773.033272:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534526464/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772773.033283:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534527104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772773.033294:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772773.033296:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772773.033302:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772773.033305:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772773.033306:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772773.033315:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534526464:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772773.033317:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1768 00000100:00080000:1.0:1790772773.033325:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772773.033332:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772773.033337:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772773.033348:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534527232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772773.033359:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534526592:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772773.033360:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772773.033372:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772773.033375:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772773.033381:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1768 00000100:00080000:1.0:1790772773.033386:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772773.033393:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772773.033398:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:2.0:1790772773.033404:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772773.033414:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534527360/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772773.033425:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772773/real 1790772773] req@ffff9c4efac4c700 x1877759534526592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772773.033449:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772773.033455:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1790772773.033459:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534526592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772773.033460:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772773.033472:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1768 00000100:00080000:1.0:1790772773.033480:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772773.033513:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772773.033519:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:2.0:1790772773.033524:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772773.033533:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790772773.033536:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534527488/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772773.033542:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534526592:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772773.033562:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534526720:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772773.033619:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772773.033635:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772773/real 1790772773] req@ffff9c4efac4ed80 x1877759534526720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772773.033652:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534526720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772773.033709:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772773.033714:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772773.033718:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534526720:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772773.033724:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534526848:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772773.033748:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772773.033760:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772773/real 1790772773] req@ffff9c4efac4dc00 x1877759534526848/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772773.033772:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534526848/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772773.033783:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772773.033787:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772773.033790:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534526848:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772773.033796:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534526976:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772773.033812:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772773.033821:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772773/real 1790772773] req@ffff9c4efac4f800 x1877759534526976/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772773.033840:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534526976/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772773.033850:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772773.033853:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772773.033856:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534526976:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772773.033862:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534527104:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772773.033877:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772773.033887:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772773/real 1790772773] req@ffff9c4efac4f480 x1877759534527104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772773.033898:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534527104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772773.033915:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772773.033920:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772773.033925:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534527104:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772773.033958:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534527232:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772773.033991:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772773.034009:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772773/real 1790772773] req@ffff9c4efac4e680 x1877759534527232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772773.034022:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534527232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772773.034037:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772773.034041:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772773.034045:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534527232:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772773.034051:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534527360:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772773.034071:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772773.034085:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772773/real 1790772773] req@ffff9c4efac4ea00 x1877759534527360/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772773.034103:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534527360/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772773.034121:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772773.034126:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772773.034132:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534527360:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772773.034143:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534527488:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772773.034171:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772773.034184:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772773/real 1790772773] req@ffff9c4efac4e300 x1877759534527488/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772773.034197:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534527488/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772773.034207:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772773.034211:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772773.034215:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534527488:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772773.096775:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772773.096781:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772773.096797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772773.096800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772773.160781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772773.160784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772773.160898:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772773.288760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772773.288760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772773.352723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772773.416804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772773.480772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772773.480829:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772773.480829:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772773.544715:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772773.608746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772773.608765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772773.608765:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772773.736742:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772773.736759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772773.736770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772773.736771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772773.800750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772773.800751:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772773.800758:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772773.800781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772773.800820:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772773.800830:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772773.928758:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772773.992731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772774.056745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772774.120774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772774.120818:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772774.120848:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772774.120886:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772774.184797:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772774.184803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772774.184987:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772774.312721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772774.312731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772774.376721:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772774.440724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772774.504720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772774.504728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772774.504765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772774.568715:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772774.632726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772774.632784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772774.632789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772774.760752:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772774.760752:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772774.760773:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772774.760777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772774.824712:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772774.824730:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772774.824734:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772774.824758:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772774.824765:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772774.824790:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772774.952707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772775.016708:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772775.080714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772775.144775:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772775.144775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772775.144775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772775.144784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772775.208732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772775.208747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772775.208892:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772775.336775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772775.336776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772775.400771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772775.464742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772775.528739:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772775.528746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772775.528804:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772775.592776:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772775.656757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772775.656819:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772775.656832:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772775.784743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772775.784795:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772775.784797:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772775.784805:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772775.848745:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772775.848747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772775.848835:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772775.848837:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772775.848855:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772775.848855:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772775.976749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772776.040772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772776.104780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772776.168748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772776.168748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772776.168749:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772776.168820:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772776.232760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772776.232789:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772776.232844:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772776.360732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772776.360740:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772776.424731:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772776.488892:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772776.552778:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772776.552790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772776.552797:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772776.616774:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772776.680719:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772776.680799:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772776.680803:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772776.808857:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772776.808870:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772776.808871:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772776.808876:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772776.872752:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772776.872790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772776.872856:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772776.872888:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772776.872896:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772776.872913:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772777.000764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772777.064749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772777.128768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772777.192794:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772777.192832:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772777.192832:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772777.192844:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772777.256835:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772777.256835:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772777.256856:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772777.384775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772777.384837:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772777.448773:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772777.512752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772777.576810:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772777.576840:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772777.576843:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772777.640793:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772777.704787:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772777.704846:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772777.704846:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772777.832777:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772777.832781:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772777.832805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772777.832824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772777.896799:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772777.896800:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772777.896806:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772777.896853:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772777.896915:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772777.896921:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772778.024778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772778.088744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790772778.152796:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772778.152805:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772778.152807:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772778.152815:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772778.152827:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1774 00000100:00080000:1.0:1790772778.152840:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772778.152853:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772778.152898:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534527616/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772778.152931:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772778.152936:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772778.152938:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772778.152942:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1774 00000100:00080000:1.0:1790772778.152946:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772778.152950:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772778.152960:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772778.152973:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534527744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772778.152983:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772778.152986:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772778.152988:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772778.152992:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1774 00000100:00080000:1.0:1790772778.152995:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772778.152999:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772778.153004:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772778.153014:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534527872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772778.153024:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772778.153028:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772778.153031:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772778.153035:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1774 00000100:00080000:1.0:1790772778.153039:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772778.153044:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772778.153050:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772778.153066:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534528000/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772778.153068:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534527616:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772778.153105:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772778.153108:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772778.153110:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772778.153119:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1774 00000100:00080000:1.0:1790772778.153123:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772778.153131:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772778.153137:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772778.153148:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534528128/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772778.153158:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772778.153163:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772778.153166:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772778.153171:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1774 00000100:00080000:1.0:1790772778.153175:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772778.153179:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772778.153184:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772778.153195:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534528256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772778.153207:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790772778.153215:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772778.153220:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772778.153225:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772778.153234:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1774 00000100:00080000:1.0:1790772778.153241:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772778.153250:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772778.153257:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772778.153279:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534528384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772778.153293:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772778.153298:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772778.153306:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772778.153308:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772778/real 1790772778] req@ffff9c4efac4fb80 x1877759534527616/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772778.153316:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1774 00000100:00080000:1.0:1790772778.153320:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772778.153325:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772778.153330:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790772778.153345:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534528512/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772778.153408:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772778.153411:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534527616/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772778.153419:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772778.153429:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772778.153450:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1774 00000100:00080000:1.0:1790772778.153455:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772778.153457:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772778.153466:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772778.153474:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790772778.153479:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772778.153486:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534527616:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772778.153503:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534528640/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772778.153504:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534527744:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772778.153564:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772778.153584:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772778/real 1790772778] req@ffff9c4efac4e300 x1877759534527744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772778.153601:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534527744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772778.153626:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772778.153631:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772778.153636:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534527744:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772778.153645:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534527872:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772778.153715:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772778.153730:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772778/real 1790772778] req@ffff9c4efac4ea00 x1877759534527872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772778.153747:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534527872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772778.153759:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772778.153764:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772778.153768:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534527872:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772778.153812:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534528000:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772778.153835:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772778.153849:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772778/real 1790772778] req@ffff9c4efac4e680 x1877759534528000/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772778.153864:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534528000/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772778.153877:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772778.153881:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772778.153886:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534528000:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772778.153894:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534528128:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772778.153914:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772778.153926:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772778/real 1790772778] req@ffff9c4efac4f480 x1877759534528128/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772778.153951:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534528128/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772778.153963:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772778.153967:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772778.153971:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534528128:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772778.153979:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534528256:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772778.153999:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772778.154012:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772778/real 1790772778] req@ffff9c4efac4f800 x1877759534528256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772778.154027:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534528256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772778.154049:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772778.154053:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772778.154057:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534528256:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772778.154065:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534528384:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772778.154085:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772778.154097:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772778/real 1790772778] req@ffff9c4efac4dc00 x1877759534528384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772778.154112:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534528384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772778.154132:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772778.154137:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772778.154141:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534528384:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772778.154148:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534528512:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772778.154168:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772778.154180:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772778/real 1790772778] req@ffff9c4efac4ed80 x1877759534528512/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772778.154194:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534528512/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772778.154208:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772778.154212:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772778.154216:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534528512:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772778.154225:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534528640:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772778.154244:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772778.154256:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772778/real 1790772778] req@ffff9c4efac4c700 x1877759534528640/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772778.154271:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534528640/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772778.154283:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772778.154287:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772778.154293:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534528640:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772778.216783:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772778.216790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772778.216795:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772778.216812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772778.280770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772778.280787:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772778.280953:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772778.408749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772778.408765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772778.472740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772778.536761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772778.600766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772778.600805:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772778.600811:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772778.664742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772778.728767:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772778.728773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772778.728792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772778.856733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772778.856734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772778.856761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772778.856778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772778.920751:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772778.920771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772778.920787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772778.920809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772778.920816:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772778.920839:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772779.048730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772779.112765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772779.176714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772779.240762:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772779.240762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772779.240808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772779.240808:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772779.304744:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772779.304782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772779.304806:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772779.432770:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772779.432770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772779.496797:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772779.560785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772779.624728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772779.624729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772779.624770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772779.688716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772779.752780:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772779.752793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772779.752824:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772779.880729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772779.880730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772779.880776:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772779.880792:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772779.944757:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772779.944798:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772779.944848:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772779.944848:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772779.944891:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772779.944912:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772780.072781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772780.136773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772780.200769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772780.264807:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772780.264835:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772780.264868:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772780.264869:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772780.328793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772780.328818:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772780.328911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772780.456778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772780.456805:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772780.520772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772780.584790:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772780.648815:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772780.648826:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772780.648833:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772780.712770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772780.776785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772780.776800:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772780.776807:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772780.904765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772780.904785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772780.904802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772780.904822:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772780.968792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772780.968809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772780.968830:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772780.968833:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772780.968873:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772780.968911:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772781.096758:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772781.160732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772781.224784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772781.288767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772781.288773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772781.288813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772781.288814:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772781.352781:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772781.352796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772781.353070:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772781.480742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772781.480795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772781.544763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772781.608785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772781.672791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772781.672795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772781.672813:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772781.736760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772781.800792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772781.800801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772781.800802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772781.928778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772781.928831:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772781.928845:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772781.928867:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772781.992786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772781.992807:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772781.992840:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772781.992842:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772781.992868:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772781.992888:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772782.120751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772782.184731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772782.248771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772782.312826:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772782.312830:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772782.312830:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772782.312832:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772782.376725:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772782.376804:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772782.377029:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772782.504749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772782.504806:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772782.568787:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772782.632758:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772782.696785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772782.696833:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772782.696833:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772782.760762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772782.824774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772782.824822:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772782.824889:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772782.952781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772782.952784:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772782.952793:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772782.952800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772783.016755:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772783.016806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772783.016888:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772783.016890:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772783.016891:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772783.016905:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772783.144756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772783.208754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772783.272790:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772783.272804:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772783.272814:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772783.272821:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772783.272838:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1779 00000100:00080000:3.0:1790772783.272855:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772783.272874:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772783.272936:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534528768/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772783.272978:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772783.272984:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772783.272989:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772783.272996:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1779 00000100:00080000:3.0:1790772783.273000:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772783.273006:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772783.273021:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772783.273042:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534528896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772783.273057:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772783.273062:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772783.273065:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772783.273072:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1779 00000100:00080000:3.0:1790772783.273076:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772783.273082:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772783.273090:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772783.273097:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534528768:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772783.273114:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534529024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772783.273129:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772783.273135:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772783.273138:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772783.273143:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1779 00000100:00080000:3.0:1790772783.273149:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772783.273155:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772783.273162:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772783.273176:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534529152/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772783.273188:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772783.273194:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772783.273197:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772783.273204:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1779 00000100:00080000:3.0:1790772783.273210:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772783.273215:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772783.273222:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:2.0:1790772783.273230:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772783.273244:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534529280/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772783.273276:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772783.273281:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772783.273307:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772783.273314:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1779 00000100:00080000:3.0:1790772783.273320:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772783.273327:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772783.273337:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772783.273353:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772783/real 1790772783] req@ffff9c4ef73c0e00 x1877759534528768/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772783.273360:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534529408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772783.273404:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772783.273409:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772783.273412:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772783.273419:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534528768/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772783.273426:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1779 00000100:00080000:3.0:1790772783.273436:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772783.273448:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772783.273452:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790772783.273455:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772783.273464:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772783.273473:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534528768:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772783.273478:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534529536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772783.273493:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534528896:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772783.273499:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772783.273510:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772783.273514:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772783.273521:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1779 00000100:00080000:3.0:1790772783.273526:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772783.273534:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772783.273540:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772783.273562:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534529664/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772783.273564:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772783.273582:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772783.273589:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772783.273592:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772783.273601:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1779 00000100:00080000:3.0:1790772783.273607:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772783.273610:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772783/real 1790772783] req@ffff9c4ef73c2d80 x1877759534528896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772783.273618:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772783.273631:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772783.273687:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534528896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772783.273697:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534529792/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772783.273733:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772783.273742:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772783.273751:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534528896:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772783.273801:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534529024:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772783.273834:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772783.273851:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772783/real 1790772783] req@ffff9c4ef73c1500 x1877759534529024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772783.273867:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534529024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772783.273879:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772783.273884:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772783.273889:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534529024:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772783.273896:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534529152:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772783.273916:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772783.273928:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772783/real 1790772783] req@ffff9c4ef73c0a80 x1877759534529152/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772783.273943:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534529152/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772783.273956:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772783.273960:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772783.273964:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534529152:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772783.273972:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534529280:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772783.273991:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772783.274002:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772783/real 1790772783] req@ffff9c4ef73c3800 x1877759534529280/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772783.274019:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534529280/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772783.274031:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772783.274035:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772783.274039:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534529280:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772783.274046:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534529408:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772783.274066:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772783.274078:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772783/real 1790772783] req@ffff9c4ef73c2300 x1877759534529408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772783.274094:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534529408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772783.274115:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772783.274120:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772783.274127:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534529408:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772783.274139:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534529536:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772783.274176:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772783.274201:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772783/real 1790772783] req@ffff9c4ef73c0700 x1877759534529536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772783.274227:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534529536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772783.274249:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772783.274254:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772783.274259:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534529536:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772783.274268:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534529664:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772783.274327:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772783.274344:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772783/real 1790772783] req@ffff9c4ef73c2a00 x1877759534529664/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772783.274360:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534529664/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772783.274375:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772783.274380:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772783.274384:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534529664:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772783.274392:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534529792:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772783.274414:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772783.274426:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772783/real 1790772783] req@ffff9c4ef73c1c00 x1877759534529792/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772783.274441:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534529792/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772783.274453:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772783.274457:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772783.274462:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534529792:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772783.336758:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772783.336776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772783.336786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772783.336809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772783.400794:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772783.400821:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772783.400859:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772783.528748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772783.528771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772783.592752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772783.656863:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772783.720777:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772783.720785:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772783.720842:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772783.784732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772783.848762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772783.848812:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772783.848813:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772783.976779:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772783.976780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772783.976821:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772783.976839:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772784.040809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772784.040851:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772784.040858:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772784.040858:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772784.040876:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772784.040916:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772784.168754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772784.232758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772784.296773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772784.360830:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772784.360858:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772784.360891:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772784.360891:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772784.424870:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772784.424897:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772784.425018:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772784.552781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772784.552858:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772784.616777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772784.680799:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772784.744799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772784.744811:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772784.744812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772784.808773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772784.872863:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772784.872864:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772784.872866:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772785.000812:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772785.000820:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772785.000822:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772785.000823:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772785.064806:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772785.064817:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772785.064846:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772785.064869:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772785.064889:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772785.064910:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772785.192773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772785.256765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772785.320743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772785.384778:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772785.384810:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772785.384813:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772785.384822:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772785.448796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772785.448848:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772785.448966:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772785.576782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772785.576797:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772785.640794:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772785.704818:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772785.768809:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772785.768810:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772785.768813:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772785.832744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772785.896844:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772785.896846:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772785.896847:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772786.024790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772786.024800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772786.024804:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772786.024809:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772786.088800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772786.088812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772786.088833:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772786.088843:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772786.088889:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772786.088901:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772786.216766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772786.280772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772786.344750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772786.408815:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772786.408818:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772786.408820:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772786.408856:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772786.472793:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772786.472892:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772786.472946:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772786.600772:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772786.600774:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772786.664784:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772786.728811:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772786.792755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772786.792773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772786.792787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772786.856737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772786.920787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772786.920849:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772786.920867:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772787.048806:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772787.048811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772787.048859:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772787.048866:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772787.112777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772787.112811:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772787.112831:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772787.112862:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772787.112885:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772787.112893:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772787.240738:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772787.304784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772787.368757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772787.432773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772787.432807:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772787.432816:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772787.432845:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772787.496801:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772787.496882:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772787.496908:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772787.624758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772787.624758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772787.688780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772787.752784:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772787.816777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772787.816791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772787.816831:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772787.880785:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772787.944735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772787.944779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772787.944789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772788.072752:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772788.072752:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772788.072752:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772788.072757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772788.136749:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772788.136781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772788.136808:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772788.136808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772788.136815:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772788.136864:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772788.264753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772788.328741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772788.392769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772788.392784:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772788.392795:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772788.392802:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772788.392818:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1784 00000100:00080000:3.0:1790772788.392835:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772788.392853:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772788.392916:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534529920/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772788.392959:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772788.392965:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772788.392968:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772788.392983:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1784 00000100:00080000:3.0:1790772788.392989:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772788.392995:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772788.393009:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772788.393027:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534530048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772788.393043:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772788.393048:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772788.393051:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772788.393059:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1784 00000100:00080000:3.0:1790772788.393065:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772788.393072:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772788.393080:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772788.393086:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534529920:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772788.393100:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534530176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772788.393116:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772788.393122:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772788.393125:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772788.393130:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1784 00000100:00080000:3.0:1790772788.393134:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772788.393140:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772788.393148:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772788.393164:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534530304/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772788.393178:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772788.393220:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772788.393224:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772788.393231:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1784 00000100:00080000:3.0:1790772788.393238:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772788.393246:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772788.393254:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772788.393283:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534530432/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772788.393291:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534530048:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772788.393322:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772788.393331:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772788.393337:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772788.393350:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1784 00000100:00080000:3.0:1790772788.393360:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772788.393366:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534530176:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772788.393379:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772788.393391:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772788.393407:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534530304:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772788.393419:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534530560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772788.393439:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772788.393446:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534530432:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772788.393449:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772788.393457:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772788.393467:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1784 00000100:00080000:3.0:1790772788.393478:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772788.393489:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772788.393502:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534530560:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772788.393512:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772788.393536:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534530688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772788.393556:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772788.393563:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1790772788.393569:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534530688:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772788.393572:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772788.393591:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1784 00000100:00080000:3.0:1790772788.393614:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772788.393629:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772788.393643:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772788.393715:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534530816/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772788.393741:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772788.393747:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772788.393750:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772788.393755:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1784 00000100:00080000:3.0:1790772788.393762:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772788.393771:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772788.393779:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772788.393795:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534530944/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772788.393803:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534530816:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772788.393876:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534530944:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772788.456736:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772788.456773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772788.456785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772788.456800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772788.520815:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772788.520846:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772788.520938:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772788.648753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772788.648835:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772788.712790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772788.776802:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772788.840777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772788.840782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772788.840834:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772788.904749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772788.968789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772788.968815:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772788.968818:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772789.096842:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772789.096844:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772789.096891:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772789.096891:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772789.160818:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772789.160846:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772789.160870:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772789.160891:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772789.160914:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772789.160940:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772789.288736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772789.352781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772789.416858:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772789.480770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772789.480797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772789.480824:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772789.480858:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772789.544794:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772789.544923:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772789.544978:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772789.672754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772789.672838:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772789.736755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772789.800743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772789.864721:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772789.864745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772789.864749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772789.928755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772789.992729:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772789.992735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772789.992740:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772790.120779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772790.120784:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772790.120799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772790.120800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772790.184777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772790.184780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772790.184801:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772790.184829:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772790.184864:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772790.184899:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772790.312751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772790.376744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772790.440840:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772790.504769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772790.504785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772790.504797:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772790.504797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772790.568795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772790.568836:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772790.568879:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772790.696736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772790.696738:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772790.760731:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772790.824805:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772790.888723:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772790.888729:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772790.888729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772790.952701:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772791.016768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772791.016789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772791.016791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772791.144714:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772791.144718:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772791.144772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772791.144773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772791.208749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772791.208749:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772791.208773:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772791.208774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772791.208824:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772791.208860:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772791.336708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772791.400723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772791.464903:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:1.0:1790772791.464922:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:1.0:1790772791.464959:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1790772791.464973:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772791.465014:0:2384:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:1.0:1790772791.465025:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772791.465038:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772791.465050:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790772791.465053:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772788/real 1790772791] req@ffff9c4ef73c2d80 x1877759534530944/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790772791.465063:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772791.465077:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790772791.465087:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534530944/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790772791.465094:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772791.465111:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:2.0:1790772791.465113:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772791.465122:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772791.465142:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534530944:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772791.465152:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772788/real 1790772791] req@ffff9c4ef73c1500 x1877759534530816/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772791.465168:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534530816/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772791.465186:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772791.465190:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772791.465195:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534530816:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772791.465201:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772788/real 1790772791] req@ffff9c4ef73c0a80 x1877759534530688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772791.465212:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534530688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772791.465222:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772791.465226:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772791.465230:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534530688:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772791.465238:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772788/real 1790772791] req@ffff9c4ef73c3800 x1877759534530560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772791.465253:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534530560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772791.465270:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772791.465275:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772791.465279:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534530560:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772791.465286:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772788/real 1790772791] req@ffff9c4ef73c2a00 x1877759534530176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772791.465299:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534530176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772791.465309:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772791.465327:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772791.465331:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534530176:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772791.465339:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772788/real 1790772791] req@ffff9c4ef73c0700 x1877759534530304/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772791.465353:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534530304/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772791.465364:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772791.465368:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772791.465372:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534530304:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772791.465379:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772788/real 1790772791] req@ffff9c4ef73c2300 x1877759534530432/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772791.465391:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534530432/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772791.465400:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772791.465403:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772791.465407:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534530432:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772791.465415:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772788/real 1790772791] req@ffff9c4ef73c0000 x1877759534529920/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772791.465429:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534529920/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772791.465439:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772791.465444:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772791.465448:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534529920:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772791.465456:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772788/real 1790772791] req@ffff9c4ef73c1c00 x1877759534530048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772791.465475:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534530048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772791.465490:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772791.465494:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772791.465498:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534530048:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772791.528755:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772791.528780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772791.528821:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772791.528828:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772791.592802:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772791.592841:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772791.592918:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772791.720810:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772791.720811:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772791.784721:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772791.848786:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772791.912794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772791.912801:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772791.912803:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772791.976790:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772792.040791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772792.040854:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772792.040858:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772792.168719:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772792.168724:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772792.168743:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772792.168749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772792.232703:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772792.232729:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772792.232746:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772792.232776:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772792.232777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772792.232781:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772792.360715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772792.424742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772792.488845:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772792.552734:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772792.552739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772792.552742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772792.552756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772792.616795:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772792.616827:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772792.616880:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772792.744761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772792.744769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772792.808716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772792.872759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772792.936768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772792.936796:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772792.936825:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772793.000749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772793.064901:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772793.064901:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772793.064907:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772793.192783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772793.192798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772793.192800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772793.192807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772793.256784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772793.256797:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772793.256799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772793.256809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772793.256853:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772793.256862:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772793.384759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772793.448739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772793.512774:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772793.512789:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772793.512798:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772793.512818:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1789 00000100:00080000:3.0:1790772793.512839:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772793.512862:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772793.512899:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772793.512953:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534531072/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772793.513010:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772793.513018:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772793.513023:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772793.513032:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1789 00000100:00080000:3.0:1790772793.513039:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772793.513052:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772793.513108:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772793.513115:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534531072:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772793.513141:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534531200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772793.513160:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772793.513168:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772793.513173:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772793.513183:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1789 00000100:00080000:3.0:1790772793.513207:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772793.513221:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772793.513232:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790772793.513240:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772793.513276:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534531328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772793.513294:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772793.513300:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772793.513304:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772793.513311:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1789 00000100:00080000:3.0:1790772793.513319:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772793.513323:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772793/real 1790772793] req@ffff9c4ef73c0e00 x1877759534531072/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772793.513350:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772793.513365:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772793.513373:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534531072/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772793.513399:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534531456/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772793.513438:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772793.513439:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772793.513455:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772793.513458:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772793.513463:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772793.513476:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534531072:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772793.513484:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1789 00000100:00080000:3.0:1790772793.513500:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772793.513514:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772793.513525:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772793.513552:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534531584/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772793.513558:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534531200:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772793.513595:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772793.513604:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772793.513609:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772793.513619:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1789 00000100:00080000:3.0:1790772793.513630:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772793.513636:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772793.513723:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772793.513736:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772793/real 1790772793] req@ffff9c4ef73c1c00 x1877759534531200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772793.513740:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772793.513780:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534531712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772793.513784:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534531200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772793.513844:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772793.513854:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772793.513859:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772793.513861:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772793.513878:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772793.513878:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1789 00000100:00100000:2.0:1790772793.513895:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534531200:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772793.513897:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772793.513920:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772793.513922:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534531328:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772793.513940:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772793.513976:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534531840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772793.513995:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772793.514014:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772793.514022:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772793.514027:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772793.514032:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772793/real 1790772793] req@ffff9c4ef73c0000 x1877759534531328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772793.514111:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534531328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772793.514132:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772793.514140:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772793.514148:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534531328:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772793.514162:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534531456:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772793.514210:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772793.514237:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772793/real 1790772793] req@ffff9c4ef73c2300 x1877759534531456/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772793.514264:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534531456/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772793.514295:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772793.514302:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772793.514310:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534531456:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772793.514355:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534531584:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772793.514403:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772793.514432:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772793/real 1790772793] req@ffff9c4ef73c0700 x1877759534531584/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772793.514460:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534531584/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772793.514492:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772793.514501:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772793.514508:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534531584:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772793.514525:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534531712:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772793.514581:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1789 00000800:00000100:2.0:1790772793.514584:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772793.514600:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772793.514638:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772793.514640:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772793/real 1790772793] req@ffff9c4ef73c2a00 x1877759534531712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772793.514657:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772793.514723:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534531712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772793.514737:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534531968/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772793.514762:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772793.514778:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772793.514793:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534531712:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772793.514794:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772793.514811:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1790772793.514819:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534531840:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772793.514820:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772793.514840:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1789 00000100:00080000:3.0:1790772793.514850:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772793.514881:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772793.514881:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772793.514900:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772793.514919:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772793/real 1790772793] req@ffff9c4ef73c3800 x1877759534531840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772793.514933:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534532096/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772793.514957:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534531840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772793.514987:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772793.514996:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772793.515005:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534531840:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772793.515046:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534531968:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772793.515120:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772793.515147:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772793/real 1790772793] req@ffff9c4ef73c0a80 x1877759534531968/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772793.515172:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534531968/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772793.515193:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772793.515202:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772793.515209:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534531968:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772793.515222:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534532096:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772793.515264:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772793.515288:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772793/real 1790772793] req@ffff9c4ef73c1500 x1877759534532096/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772793.515311:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534532096/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772793.515340:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772793.515348:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772793.515354:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534532096:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772793.576765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772793.576791:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772793.576803:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772793.576803:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772793.640798:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772793.640825:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772793.640862:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772793.768816:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772793.768834:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772793.832748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772793.896816:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772793.960769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772793.960774:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772793.960797:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772794.024766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772794.088773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772794.088774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772794.088806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772794.216755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772794.216775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772794.216797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772794.216804:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772794.280781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772794.280781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772794.280809:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772794.280811:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772794.280857:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772794.280865:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772794.408767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772794.472782:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772794.536990:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772794.600776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772794.600796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772794.600797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772794.600840:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772794.664826:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772794.664827:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772794.664908:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772794.792802:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772794.792827:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772794.856752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772794.920798:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772794.984758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772794.984797:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772794.984808:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772795.048826:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772795.112756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772795.112773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772795.112776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772795.240748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772795.240759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772795.240785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772795.240805:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772795.304771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772795.304795:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772795.304825:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772795.304827:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772795.304870:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772795.304876:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772795.432759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772795.496766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772795.560754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772795.624796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772795.624812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772795.624813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772795.624846:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772795.688755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772795.688856:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772795.688967:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772795.816774:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772795.816791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772795.880768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772795.944854:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772796.008775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772796.008787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772796.008805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772796.072767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772796.136779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772796.136801:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772796.136828:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772796.264776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772796.264787:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772796.264802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772796.264861:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772796.328764:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772796.328769:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772796.328856:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772796.328874:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772796.328912:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772796.328931:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772796.456750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772796.520725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772796.584741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772796.648737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772796.648744:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772796.648751:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772796.648754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772796.712717:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772796.712771:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772796.712877:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772796.840711:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772796.840723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772796.904745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772796.968729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772797.032727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772797.032755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772797.032756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772797.096739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772797.160752:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772797.160752:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772797.160776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772797.288772:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772797.288794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772797.288798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772797.288846:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772797.352800:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772797.352854:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772797.352862:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772797.352863:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772797.352892:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772797.352956:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772797.480758:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772797.544751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772797.608755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772797.672778:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772797.672812:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772797.672818:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772797.672825:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772797.736823:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772797.736880:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772797.736910:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772797.864748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772797.864779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772797.928784:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772797.992859:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772798.056737:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772798.056765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772798.056767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772798.120753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772798.184741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772798.184771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772798.184809:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772798.312766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772798.312766:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772798.312787:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772798.312794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772798.376854:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772798.376863:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772798.376863:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772798.376868:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772798.376948:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772798.377016:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772798.504723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772798.568757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772798.632728:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772798.632797:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772798.632808:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772798.632817:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772798.632834:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1794 00000100:00080000:1.0:1790772798.632852:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772798.632872:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772798.632935:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534532224/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772798.633004:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772798.633011:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772798.633015:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772798.633022:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1794 00000100:00080000:1.0:1790772798.633029:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772798.633039:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772798.633058:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772798.633076:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534532224:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772798.633087:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534532352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772798.633109:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772798.633116:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772798.633120:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772798.633128:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1794 00000100:00080000:1.0:1790772798.633135:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772798.633144:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772798.633152:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772798.633170:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534532480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772798.633178:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772798.633218:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772798.633225:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772798.633229:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772798.633239:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772798/real 1790772798] req@ffff9c4efac4df80 x1877759534532224/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772798.633241:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1794 00000100:00080000:1.0:1790772798.633252:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772798.633265:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772798.633270:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534532224/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772798.633276:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00080000:2.0:1790772798.633287:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772798.633294:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772798.633300:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534532224:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772798.633306:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534532608/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772798.633324:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772798.633331:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772798.633335:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772798.633340:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534532352:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772798.633348:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1794 00000100:00080000:1.0:1790772798.633355:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772798.633366:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772798.633373:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772798.633381:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772798.633391:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772798/real 1790772798] req@ffff9c4efac4c700 x1877759534532352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772798.633407:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534532736/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772798.633409:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534532352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772798.633452:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772798.633460:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772798.633464:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772798.633466:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534532352:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772798.633480:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1790772798.633482:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534532480:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772798.633485:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772798.633497:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1794 00000800:00000100:2.0:1790772798.633506:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772798.633507:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772798.633520:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772798/real 1790772798] req@ffff9c4efac4ed80 x1877759534532480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772798.633523:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772798.633540:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772798.633545:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534532480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772798.633558:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772798.633564:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790772798.633570:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534532864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772798.633573:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534532480:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772798.633593:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534532608:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790772798.633600:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772798.633608:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772798.633612:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772798.633622:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1794 00000100:00080000:1.0:1790772798.633630:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772798.633637:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772798.633644:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772798.633695:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772798.633713:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772798/real 1790772798] req@ffff9c4efac4dc00 x1877759534532608/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772798.633721:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534532992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772798.633736:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534532608/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772798.633751:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772798.633755:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772798.633761:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772798.633763:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772798.633769:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772798.633771:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534532608:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790772798.633788:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1794 00000100:00080000:1.0:1790772798.633796:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772798.633805:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534532736:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790772798.633811:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772798.633822:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:2.0:1790772798.633838:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772798.633849:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534533120/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772798.633856:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772798/real 1790772798] req@ffff9c4efac4f800 x1877759534532736/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772798.633907:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534532736/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772798.633909:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772798.633923:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772798.633929:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772798.633944:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1794 00000100:00080000:2.0:1790772798.633948:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772798.633953:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772798.633956:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772798.633964:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534532736:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790772798.633971:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772798.633975:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534532864:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790772798.634006:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:2.0:1790772798.634030:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772798.634052:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534533248/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772798.634056:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772798/real 1790772798] req@ffff9c4efac4f480 x1877759534532864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772798.634098:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534532864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772798.634120:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772798.634125:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772798.634129:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534532864:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772798.634136:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534532992:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772798.634159:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772798.634170:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772798/real 1790772798] req@ffff9c4efac4e680 x1877759534532992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772798.634182:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534532992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772798.634190:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772798.634193:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772798.634198:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534532992:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772798.634213:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534533120:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772798.634230:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772798.634240:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772798/real 1790772798] req@ffff9c4efac4ea00 x1877759534533120/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772798.634252:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534533120/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772798.634262:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772798.634265:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772798.634269:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534533120:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772798.634274:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534533248:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772798.634289:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772798.634298:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772798/real 1790772798] req@ffff9c4efac4e300 x1877759534533248/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772798.634310:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534533248/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772798.634319:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772798.634322:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772798.634325:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534533248:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772798.696818:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772798.696820:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772798.696821:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772798.696824:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772798.760805:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772798.760935:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772798.760939:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772798.888797:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772798.888843:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772798.952793:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772799.016799:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772799.080786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772799.080790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772799.080825:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772799.144782:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772799.208771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772799.208795:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772799.208795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772799.336766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772799.336838:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772799.336838:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772799.336854:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772799.400767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772799.400767:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772799.400784:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772799.400801:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772799.400839:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772799.400851:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772799.528775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772799.592742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772799.656774:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772799.720800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772799.720819:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772799.720819:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772799.720868:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772799.784884:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772799.784886:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772799.785019:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772799.912760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772799.912775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772799.976713:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772800.040718:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772800.104771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772800.104790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772800.104790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772800.168765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772800.232720:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772800.232735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772800.232736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772800.360755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772800.360757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772800.360770:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772800.360812:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772800.424760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772800.424800:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772800.424800:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772800.424820:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772800.424823:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772800.424847:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772800.552711:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772800.616791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772800.680732:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772800.744734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772800.744742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772800.744807:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772800.744859:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772800.808814:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772800.808904:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772800.808960:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772800.936750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772800.936785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772801.000803:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772801.064706:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772801.128722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772801.128725:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772801.128727:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772801.192710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772801.256744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772801.256746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772801.256746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772801.384735:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772801.384741:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772801.384786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772801.384788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772801.448709:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772801.448751:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772801.448774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772801.448778:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772801.448803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772801.448815:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772801.576796:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772801.640718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772801.704806:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772801.768741:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772801.768741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772801.768745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772801.768847:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772801.832765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772801.832867:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772801.832884:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772801.960730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772801.960744:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772802.024781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772802.088747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772802.152774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772802.152808:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772802.152821:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772802.216801:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772802.280787:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772802.280788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772802.280826:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772802.408822:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772802.408823:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772802.408824:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772802.408880:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772802.472769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772802.472826:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772802.472848:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772802.472854:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772802.472877:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772802.472901:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772802.600748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772802.664751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772802.728806:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772802.792762:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772802.792764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772802.792777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772802.792795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772802.856806:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772802.856858:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772802.856859:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772802.984772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772802.984789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772803.048742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772803.112743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772803.176752:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772803.176787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772803.176804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772803.240758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772803.304743:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772803.304781:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772803.304785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772803.432764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772803.432776:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772803.432800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772803.432819:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772803.496800:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772803.496815:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772803.496838:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772803.496856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772803.496878:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772803.496930:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772803.624754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772803.688764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772803.752775:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772803.752871:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772803.752911:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772803.752921:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772803.752942:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1799 00000100:00080000:2.0:1790772803.752961:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772803.752981:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772803.753050:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534533376/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772803.753111:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772803.753117:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772803.753120:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772803.753127:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1799 00000100:00080000:2.0:1790772803.753132:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772803.753139:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772803.753154:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772803.753173:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534533504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772803.753189:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772803.753198:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1790772803.753200:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534533376:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772803.753202:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772803.753218:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1799 00000100:00080000:2.0:1790772803.753224:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772803.753230:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772803.753238:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772803.753259:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534533632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772803.753271:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772803.753277:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772803.753280:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772803.753286:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1799 00000100:00080000:2.0:1790772803.753305:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772803.753312:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772803.753320:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772803.753327:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772803.753344:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534533760/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772803.753360:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772803.753365:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772803.753368:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772803.753374:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1799 00000100:00080000:2.0:1790772803.753380:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772803.753387:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772803.753394:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772803.753401:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772803/real 1790772803] req@ffff9c4efbcc2a00 x1877759534533376/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772803.753417:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534533888/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772803.753437:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772803.753445:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1790772803.753446:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534533376/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772803.753452:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772803.753468:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1799 00000100:00080000:2.0:1790772803.753480:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772803.753486:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772803.753489:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772803.753499:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1790772803.753501:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772803.753515:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534533376:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772803.753524:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534534016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772803.753538:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772803.753543:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772803.753546:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772803.753553:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1799 00000100:00080000:2.0:1790772803.753558:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772803.753563:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772803.753571:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772803.753577:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534533504:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772803.753590:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534534144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772803.753609:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772803.753614:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772803.753618:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772803.753624:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1799 00000100:00080000:2.0:1790772803.753631:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772803.753638:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:3.0:1790772803.753688:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790772803.753703:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772803.753728:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534534272/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772803.753735:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772803/real 1790772803] req@ffff9c4efbcc1500 x1877759534533504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772803.753759:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772803.753775:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772803.753787:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772803.753800:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1799 00000100:00100000:3.0:1790772803.753802:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534533504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772803.753815:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772803.753835:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772803.753840:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790772803.753848:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:3.0:1790772803.753851:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772803.753864:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534533504:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772803.753877:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534533632:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772803.753911:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534534400/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772803.753956:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772803.753977:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772803/real 1790772803] req@ffff9c4efbcc1c00 x1877759534533632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772803.753994:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534533632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772803.754007:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772803.754012:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772803.754017:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534533632:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772803.754029:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534533760:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772803.754068:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772803.754093:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772803/real 1790772803] req@ffff9c4efbcc1880 x1877759534533760/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772803.754119:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534533760/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772803.754140:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772803.754145:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772803.754151:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534533760:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772803.754160:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534533888:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772803.754191:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772803.754206:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772803/real 1790772803] req@ffff9c4efbcc2300 x1877759534533888/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772803.754223:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534533888/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772803.754237:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772803.754241:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772803.754246:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534533888:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772803.754253:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534534016:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772803.754273:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772803.754285:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772803/real 1790772803] req@ffff9c4efbcc2680 x1877759534534016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772803.754300:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534534016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772803.754322:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772803.754325:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772803.754330:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534534016:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772803.754359:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534534144:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772803.754381:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772803.754393:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772803/real 1790772803] req@ffff9c4efbcc0700 x1877759534534144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772803.754408:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534534144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772803.754429:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772803.754433:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772803.754438:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534534144:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772803.754445:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534534272:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772803.754464:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772803.754475:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772803/real 1790772803] req@ffff9c4efbcc1f80 x1877759534534272/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772803.754489:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534534272/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772803.754503:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772803.754507:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772803.754512:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534534272:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772803.754520:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534534400:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772803.754553:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772803.754575:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772803/real 1790772803] req@ffff9c4efbcc0a80 x1877759534534400/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772803.754602:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534534400/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772803.754622:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772803.754630:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772803.754639:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534534400:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772803.816773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772803.816773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772803.816791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772803.816791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772803.880789:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772803.880836:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772803.880941:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772804.008788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772804.008795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772804.072770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772804.136733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772804.200779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772804.200806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772804.200820:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772804.264794:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772804.328773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772804.328810:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772804.328814:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772804.456795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772804.456795:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772804.456820:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772804.456835:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772804.520792:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772804.520820:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772804.520846:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772804.520898:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772804.520899:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772804.520940:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772804.648784:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772804.712778:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772804.776785:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772804.840772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772804.840776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772804.840784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772804.840796:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772804.904767:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772804.904816:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772804.904975:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772805.032787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772805.032801:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772805.096800:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772805.160919:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772805.224741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772805.224741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772805.224797:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772805.288750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772805.352715:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772805.352783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772805.352784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772805.480714:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772805.480752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772805.480764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772805.480781:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772805.544718:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772805.544728:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772805.544764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772805.546125:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772805.546144:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772805.546182:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772805.672728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772805.736692:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772805.800708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772805.864710:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772805.864720:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772805.864726:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772805.864788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772805.928714:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772805.928804:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772805.928834:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772806.056706:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772806.056780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772806.120723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772806.184695:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772806.248713:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772806.248720:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772806.248731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772806.312744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772806.376750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772806.376750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772806.376751:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772806.504720:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772806.504725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772806.504729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772806.504777:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772806.568718:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772806.568719:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772806.568730:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772806.568819:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772806.568840:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772806.568884:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772806.696716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772806.760765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772806.824781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772806.888732:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772806.888774:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772806.888775:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772806.888849:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772806.952802:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772806.952934:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772806.952938:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772807.080800:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772807.080811:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772807.144764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772807.208759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772807.272745:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772807.272765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772807.272795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772807.336808:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772807.400759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772807.400782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772807.400789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772807.528734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772807.528745:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772807.528770:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772807.528799:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772807.592727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772807.592735:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772807.592749:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772807.592789:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772807.592802:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772807.592808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772807.720724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772807.784746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772807.848780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772807.912794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772807.912836:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772807.912842:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772807.912887:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772807.976796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772807.976841:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772807.976916:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772808.104777:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772808.104777:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772808.168771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772808.232760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772808.296722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772808.296779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772808.296794:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772808.360836:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772808.424789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772808.424823:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772808.424877:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772808.552861:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772808.552871:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772808.552871:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772808.552872:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772808.616856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772808.616856:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772808.616874:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772808.616886:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772808.616935:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772808.616969:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772808.744729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772808.808744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772808.872853:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772808.872879:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772808.872889:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772808.872895:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772808.872908:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1804 00000100:00080000:3.0:1790772808.872920:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772808.872934:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772808.872976:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534534528/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772808.873019:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772808.873024:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772808.873026:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772808.873031:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1804 00000100:00080000:3.0:1790772808.873035:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772808.873042:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772808.873054:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772808.873073:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772808.873086:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772808.873091:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772808.873093:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772808.873098:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1804 00000100:00080000:3.0:1790772808.873102:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772808.873111:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772808.873125:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772808.873126:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534534528:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772808.873150:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772808.873176:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772808.873181:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772808.873184:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772808.873189:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1804 00000100:00080000:3.0:1790772808.873195:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772808.873203:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772808.873211:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772808.873225:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534534912/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772808.873238:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772808.873243:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772808.873245:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772808.873251:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1804 00000800:00000100:2.0:1790772808.873264:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772808.873268:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772808.873283:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772808.873295:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772808.873324:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534535040/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772808.873342:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772808/real 1790772808] req@ffff9c4ef73c2d80 x1877759534534528/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772808.873348:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772808.873375:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772808.873382:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772808.873385:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534534528/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772808.873392:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1804 00000100:00080000:3.0:1790772808.873406:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772808.873419:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772808.873441:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790772808.873447:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790772808.873462:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534535168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772808.873465:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772808.873498:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534534528:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772808.873501:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772808.873527:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772808.873533:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772808.873542:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1804 00000100:00080000:3.0:1790772808.873548:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772808.873554:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772808.873559:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534534656:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772808.873562:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772808.873596:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534535296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772808.873611:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772808.873617:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772808.873619:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772808.873626:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1804 00000100:00080000:3.0:1790772808.873630:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772808.873638:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772808.873647:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772808.873707:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772808.873721:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772808/real 1790772808] req@ffff9c4ef73c1500 x1877759534534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772808.873731:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534535424/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772808.873819:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772808.873823:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772808.873827:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772808.873830:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534534656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772808.873839:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1804 00000100:00080000:3.0:1790772808.873852:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772808.873873:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772808.873886:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:2.0:1790772808.873889:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772808.873900:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772808.873907:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534535552/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772808.873910:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534534656:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772808.873987:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534534784:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772808.874025:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772808.874042:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772808/real 1790772808] req@ffff9c4ef73c0a80 x1877759534534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772808.874060:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534534784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772808.874073:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772808.874078:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772808.874083:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534534784:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772808.874091:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534534912:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772808.874114:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772808.874127:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772808/real 1790772808] req@ffff9c4ef73c3800 x1877759534534912/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772808.874145:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534534912/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772808.874159:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772808.874163:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772808.874167:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534534912:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772808.874176:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534535040:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772808.874196:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772808.874209:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772808/real 1790772808] req@ffff9c4ef73c2a00 x1877759534535040/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772808.874225:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534535040/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772808.874238:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772808.874252:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772808.874256:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534535040:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772808.874264:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534535168:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772808.874285:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772808.874298:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772808/real 1790772808] req@ffff9c4ef73c0700 x1877759534535168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772808.874313:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534535168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772808.874343:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772808.874348:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772808.874353:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534535168:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772808.874362:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534535296:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772808.874389:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772808.874412:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772808/real 1790772808] req@ffff9c4ef73c2300 x1877759534535296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772808.874438:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534535296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772808.874457:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772808.874465:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772808.874473:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534535296:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772808.874488:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534535424:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772808.874522:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772808.874539:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772808/real 1790772808] req@ffff9c4ef73c0000 x1877759534535424/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772808.874556:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534535424/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772808.874570:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772808.874575:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772808.874581:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534535424:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772808.874590:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534535552:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772808.874611:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772808.874629:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772808/real 1790772808] req@ffff9c4ef73c1c00 x1877759534535552/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772808.874644:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534535552/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772808.874696:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772808.874702:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772808.874708:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534535552:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772808.936828:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772808.936830:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772808.936836:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772808.936850:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772809.000810:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772809.000820:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772809.000905:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772809.128777:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772809.128819:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772809.192757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772809.256744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772809.320743:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772809.320825:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772809.320885:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772809.384744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772809.448753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772809.448795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772809.448824:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772809.576762:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772809.576788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772809.576823:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772809.576855:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772809.640807:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772809.640812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772809.640852:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772809.640866:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772809.640888:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772809.640896:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772809.768801:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772809.832801:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772809.896803:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772809.960754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772809.960781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772809.960815:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772809.960828:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772810.024813:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772810.024895:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772810.024916:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772810.152742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772810.152822:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772810.216815:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772810.280857:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772810.344737:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772810.344798:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772810.344799:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772810.408777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772810.472782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772810.472784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772810.472790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772810.600740:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772810.600807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772810.600824:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772810.600831:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772810.664793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772810.664819:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772810.664823:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772810.664836:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772810.664869:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772810.664883:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772810.792829:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772810.856733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772810.920798:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772810.984770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772810.984770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772810.984777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772810.984777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772811.048815:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772811.048859:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772811.048877:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772811.176809:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772811.176842:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772811.240783:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772811.304838:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772811.368778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772811.368786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772811.368851:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772811.432789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772811.496771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772811.496807:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772811.496856:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772811.624829:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772811.624835:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772811.624842:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772811.624888:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772811.688786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772811.688821:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772811.688883:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772811.688894:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772811.688925:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772811.688976:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772811.816851:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772811.880809:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772811.944814:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772812.008792:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772812.008802:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772812.008841:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772812.008857:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772812.072802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772812.072885:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772812.072915:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772812.200785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772812.200799:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772812.264815:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772812.328936:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772812.392742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772812.392813:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772812.392843:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772812.456774:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772812.520787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772812.520787:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772812.520787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772812.648822:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772812.648825:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772812.648844:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772812.648845:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772812.712764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772812.712764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772812.712838:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772812.712861:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772812.712865:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772812.712876:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772812.840810:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772812.904743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772812.968843:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772813.032839:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772813.032839:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772813.032850:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772813.032889:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772813.096868:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772813.096889:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772813.096945:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772813.224771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772813.224775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772813.288770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772813.352766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772813.416787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772813.416791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772813.416832:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772813.480773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772813.544796:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772813.544796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772813.544826:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772813.672793:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772813.672829:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772813.672839:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772813.672842:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772813.736778:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772813.736829:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772813.736850:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772813.736895:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772813.736941:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772813.736944:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772813.864779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772813.928803:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772813.992787:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772813.992834:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772813.992847:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772813.992854:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772813.992871:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1809 00000100:00080000:1.0:1790772813.992888:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772813.992907:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772813.992972:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534535680/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772813.993017:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772813.993025:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772813.993029:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772813.993039:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1809 00000100:00080000:1.0:1790772813.993047:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772813.993060:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772813.993083:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772813.993086:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534535680:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772813.993124:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534535808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772813.993147:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772813.993153:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772813.993157:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772813.993164:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1809 00000100:00080000:1.0:1790772813.993174:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772813.993183:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772813.993197:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772813.993198:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772813.993223:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534535936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772813.993236:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772813.993242:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772813.993247:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772813.993255:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1809 00000100:00080000:1.0:1790772813.993262:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772813.993271:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772813.993279:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772813.993281:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772813/real 1790772813] req@ffff9c4efac4fb80 x1877759534535680/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772813.993303:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534536064/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772813.993319:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772813.993321:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534535680/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772813.993325:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772813.993329:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772813.993339:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1809 00000100:00080000:1.0:1790772813.993346:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772813.993359:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772813.993368:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:2.0:1790772813.993373:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790772813.993385:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534536192/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772813.993386:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772813.993411:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534535680:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772813.993421:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772813.993432:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772813.993437:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772813.993444:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1809 00000100:00080000:1.0:1790772813.993449:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772813.993456:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772813.993474:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534535808:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790772813.993479:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772813.993507:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534536320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772813.993523:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772813.993529:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772813.993532:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790772813.993545:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772813.993547:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1809 00000100:00080000:1.0:1790772813.993559:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772813.993568:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772813.993576:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772813.993584:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772813/real 1790772813] req@ffff9c4efac4e300 x1877759534535808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772813.993594:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534536448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772813.993714:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772813.993721:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772813.993725:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772813.993737:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534535808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772813.993739:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1809 00000100:00080000:1.0:1790772813.993750:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772813.993768:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772813.993779:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:2.0:1790772813.993790:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790772813.993800:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534536576/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772813.993804:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772813.993829:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772813.993829:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534535808:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772813.993839:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772813.993842:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772813.993859:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1809 00000100:00080000:1.0:1790772813.993866:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772813.993869:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534535936:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772813.993881:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772813.993892:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772813.993911:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534536704/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772813.993928:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772813.993951:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772813/real 1790772813] req@ffff9c4efac4ea00 x1877759534535936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772813.993966:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534535936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772813.993976:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772813.993980:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772813.993983:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534535936:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772813.993990:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534536064:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772813.994018:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772813.994036:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772813/real 1790772813] req@ffff9c4efac4e680 x1877759534536064/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772813.994055:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534536064/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772813.994070:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772813.994074:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772813.994078:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534536064:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772813.994086:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534536192:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772813.994108:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772813.994119:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772813/real 1790772813] req@ffff9c4efac4f480 x1877759534536192/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772813.994130:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534536192/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772813.994141:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772813.994146:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772813.994152:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534536192:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772813.994183:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534536320:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772813.994215:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772813.994229:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772813/real 1790772813] req@ffff9c4efac4f800 x1877759534536320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772813.994248:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534536320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772813.994263:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772813.994267:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772813.994270:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534536320:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772813.994276:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534536448:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772813.994294:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772813.994304:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772813/real 1790772813] req@ffff9c4efac4dc00 x1877759534536448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772813.994322:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534536448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772813.994335:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772813.994340:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772813.994346:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534536448:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772813.994357:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534536576:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772813.994385:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772813.994397:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772813/real 1790772813] req@ffff9c4efac4ed80 x1877759534536576/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772813.994411:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534536576/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772813.994436:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772813.994441:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772813.994447:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534536576:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772813.994457:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534536704:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772813.994486:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772813.994504:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772813/real 1790772813] req@ffff9c4efac4c700 x1877759534536704/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772813.994523:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534536704/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772813.994538:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772813.994544:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772813.994549:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534536704:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772814.056747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772814.056760:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772814.056818:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772814.056845:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772814.120756:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772814.120772:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772814.120903:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772814.248713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772814.248714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772814.312764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772814.376726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772814.440729:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772814.440734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772814.440762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772814.504738:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772814.568804:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772814.568804:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772814.568808:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772814.696798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772814.696830:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772814.696834:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772814.696850:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772814.760792:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772814.760826:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772814.760855:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772814.760880:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772814.760880:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772814.760893:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772814.888783:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772814.952758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772815.016759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772815.080805:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772815.080905:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772815.080908:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772815.080924:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772815.144789:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772815.144888:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772815.144910:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772815.272808:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772815.272811:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772815.336773:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772815.400825:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772815.464785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772815.464827:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772815.464828:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772815.528739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772815.592770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772815.592803:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772815.592804:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772815.720751:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772815.720753:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772815.720755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772815.720759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772815.784730:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772815.784753:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772815.784765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772815.784769:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772815.784777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772815.784799:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772815.912721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772815.976722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772816.040701:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772816.104741:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772816.104747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772816.104770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772816.104827:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772816.168794:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772816.168797:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772816.168827:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772816.296767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772816.296773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772816.360781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772816.424886:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772816.488778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772816.488782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772816.488805:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772816.552741:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772816.616784:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772816.616789:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772816.616815:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772816.744870:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772816.744997:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772816.745138:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772816.745755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772816.808826:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772816.808854:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772816.808861:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772816.808864:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772816.808927:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772816.808932:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772816.936773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772817.000768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772817.064818:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772817.128770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772817.128803:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772817.128817:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772817.128829:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772817.192831:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772817.192843:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772817.192975:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772817.320757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772817.320788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772817.384768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772817.448785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772817.512774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772817.512799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772817.512806:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772817.576762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772817.640797:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772817.640802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772817.640814:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772817.769018:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772817.769253:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772817.769398:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772817.769747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772817.832792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772817.832824:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772817.832825:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772817.832830:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772817.832872:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772817.832908:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772817.960759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772818.024773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772818.088759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772818.152738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772818.152788:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772818.152807:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772818.152827:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772818.216779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772818.216857:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772818.217001:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772818.344750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772818.344786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772818.408742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772818.472822:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772818.536739:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772818.536811:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772818.536813:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772818.600779:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772818.664805:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772818.664819:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772818.664823:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772818.792825:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772818.792834:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772818.792837:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772818.792863:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772818.856809:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772818.856831:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772818.856844:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772818.856857:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772818.856887:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772818.856904:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772818.984736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772819.048772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772819.112766:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772819.112869:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772819.112879:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772819.112886:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772819.112903:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1815 00000100:00080000:2.0:1790772819.112921:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772819.112941:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772819.113009:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534536832/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772819.113057:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772819.113063:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772819.113066:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772819.113072:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1815 00000100:00080000:2.0:1790772819.113077:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772819.113084:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772819.113099:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772819.113118:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534536960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772819.113135:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772819.113141:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772819.113145:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772819.113154:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534536832:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772819.113156:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1815 00000100:00080000:2.0:1790772819.113172:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772819.113182:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772819.113190:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772819.113213:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534537088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772819.113224:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772819.113228:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772819.113231:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772819.113238:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1815 00000100:00080000:2.0:1790772819.113243:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772819.113249:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772819.113256:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772819.113272:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534537216/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772819.113285:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772819.113291:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772819.113295:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772819.113300:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1815 00000100:00080000:2.0:1790772819.113307:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772819.113314:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772819.113322:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:3.0:1790772819.113335:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772819.113350:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534537344/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772819.113369:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772819.113374:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772819.113378:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772819.113386:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1815 00000100:00080000:2.0:1790772819.113395:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772819.113405:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772819.113415:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772819/real 1790772819] req@ffff9c4efbcc0380 x1877759534536832/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790772819.113421:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772819.113449:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534537472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772819.113464:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534536832/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772819.113470:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772819.113499:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772819.113511:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772819.113532:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1815 00000100:00080000:3.0:1790772819.113537:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772819.113538:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772819.113554:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772819.113560:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790772819.113565:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772819.113574:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534536832:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772819.113617:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534537600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772819.113634:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772819.113640:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772819.113643:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772819.113695:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534536960:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772819.113705:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1815 00000100:00080000:2.0:1790772819.113713:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772819.113722:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772819.113733:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772819.113764:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534537728/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772819.113765:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772819.113789:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772819.113794:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772819.113797:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772819.113806:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1815 00000100:00080000:2.0:1790772819.113815:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772819.113829:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772819.113832:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772819/real 1790772819] req@ffff9c4efbcc3100 x1877759534536960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790772819.113838:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772819.113862:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534537856/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772819.113908:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534536960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772819.113945:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772819.113955:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772819.113963:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534536960:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772819.113978:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534537088:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772819.114019:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772819.114037:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772819/real 1790772819] req@ffff9c4efbcc0a80 x1877759534537088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772819.114053:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534537088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772819.114065:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772819.114070:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772819.114075:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534537088:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772819.114083:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534537216:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772819.114104:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772819.114116:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772819/real 1790772819] req@ffff9c4efbcc1f80 x1877759534537216/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772819.114132:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534537216/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772819.114144:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772819.114149:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772819.114153:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534537216:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772819.114161:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534537344:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772819.114180:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772819.114192:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772819/real 1790772819] req@ffff9c4efbcc0700 x1877759534537344/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772819.114207:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534537344/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772819.114218:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772819.114222:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772819.114227:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534537344:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772819.114234:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534537472:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772819.114255:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772819.114267:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772819/real 1790772819] req@ffff9c4efbcc2680 x1877759534537472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772819.114282:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534537472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772819.114309:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772819.114314:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772819.114318:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534537472:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772819.114326:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534537600:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772819.114346:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772819.114358:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772819/real 1790772819] req@ffff9c4efbcc2300 x1877759534537600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772819.114372:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534537600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772819.114383:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772819.114387:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772819.114391:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534537600:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772819.114424:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534537728:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772819.114445:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772819.114457:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772819/real 1790772819] req@ffff9c4efbcc1880 x1877759534537728/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772819.114472:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534537728/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772819.114485:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772819.114489:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772819.114495:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534537728:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772819.114503:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534537856:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772819.114523:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772819.114535:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772819/real 1790772819] req@ffff9c4efbcc1c00 x1877759534537856/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772819.114549:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534537856/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772819.114561:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772819.114565:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772819.114570:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534537856:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772819.176772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772819.176779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772819.176796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772819.176826:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772819.240797:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772819.240887:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772819.240962:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772819.368727:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772819.368737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772819.432801:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772819.496772:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772819.560754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772819.560773:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772819.560794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772819.624753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772819.688767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772819.688791:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772819.688814:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772819.816903:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772819.816984:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772819.817083:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772819.817763:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772819.880783:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772819.880785:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772819.880786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772819.880820:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772819.880857:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772819.880866:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772820.008757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772820.072755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772820.136807:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772820.200791:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772820.200846:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772820.200847:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772820.200849:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772820.264811:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772820.264913:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772820.264925:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772820.392856:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772820.392880:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772820.456798:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772820.520880:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772820.584749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772820.584801:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772820.584808:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772820.648771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772820.712801:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772820.712821:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772820.712823:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772820.840749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772820.840751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772820.840794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772820.840804:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772820.904802:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772820.904810:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772820.904843:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772820.904853:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772820.904887:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772820.904893:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772821.032730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772821.096764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772821.160761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772821.224786:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772821.224799:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772821.224799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772821.224822:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772821.288793:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772821.288893:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772821.288945:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772821.416765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772821.416810:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772821.480768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772821.544831:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772821.608803:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772821.608803:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772821.608837:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772821.672745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772821.736801:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772821.736801:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772821.736831:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772821.864777:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772821.864779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772821.864842:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772821.864844:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772821.928788:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772821.928832:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772821.928839:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772821.928887:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772821.928888:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772821.928910:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772822.056760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772822.120750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772822.184754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772822.248759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772822.248795:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772822.248868:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772822.248873:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772822.312811:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772822.312884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772822.312915:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772822.440757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772822.440800:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772822.504782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772822.568777:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772822.632731:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772822.632731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772822.632733:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772822.696726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772822.760708:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772822.760732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772822.760802:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772822.888797:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772822.888813:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772822.888820:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772822.888862:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772822.952776:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772822.952797:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772822.952811:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772822.952838:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772822.952861:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772822.952899:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772823.080757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772823.144749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772823.208735:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772823.272775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772823.272859:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772823.272892:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772823.272902:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772823.336770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772823.336867:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772823.336945:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772823.464773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772823.464783:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772823.528765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772823.592833:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772823.656758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772823.656769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772823.656792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772823.720787:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772823.784777:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772823.784777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772823.784786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772823.912778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772823.912816:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772823.912860:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772823.912879:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772823.976779:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772823.976786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772823.976828:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772823.976885:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772823.976887:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772823.976887:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772824.104790:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772824.168770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790772824.232781:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772824.232797:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772824.232817:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772824.232835:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1820 00000100:00100000:0.0:1790772824.232847:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772824.232857:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772824.232881:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772824.232951:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534537984/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772824.233000:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772824.233006:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772824.233009:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772824.233015:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1820 00000100:00080000:2.0:1790772824.233031:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772824.233037:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772824.233052:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772824.233073:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534538112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772824.233094:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772824.233103:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1790772824.233105:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534537984:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772824.233108:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772824.233125:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1820 00000100:00080000:2.0:1790772824.233138:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772824.233145:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772824.233151:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772824.233165:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534538240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772824.233178:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772824.233183:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772824.233186:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772824.233193:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1820 00000100:00080000:2.0:1790772824.233216:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772824.233225:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772824.233232:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772824.233245:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772824.233249:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534538368/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772824.233266:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772824.233272:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772824.233276:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772824.233283:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1820 00000100:00080000:2.0:1790772824.233290:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772824.233296:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772824.233304:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772824.233336:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772824/real 1790772824] req@ffff9c4efbcc1500 x1877759534537984/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772824.233340:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534538496/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772824.233416:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772824.233439:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772824.233443:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772824.233450:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1820 00000100:00080000:2.0:1790772824.233459:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772824.233472:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772824.233475:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534537984/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790772824.233481:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772824.233556:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534538624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772824.233558:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772824.233580:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772824.233612:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772824.233614:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534537984:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772824.233632:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772824.233641:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772824.233695:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534538112:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772824.233703:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1820 00000100:00080000:2.0:1790772824.233712:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772824.233723:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772824.233732:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772824.233752:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534538752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772824.233811:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772824.233817:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772824.233820:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772824.233827:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1820 00000100:00080000:2.0:1790772824.233839:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772824.233846:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772824.233852:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:3.0:1790772824.233861:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772824.233875:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534538880/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772824.233889:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772824.233895:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772824.233897:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772824.233909:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772824/real 1790772824] req@ffff9c4efbcc2a00 x1877759534538112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772824.233910:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1820 00000100:00080000:2.0:1790772824.233921:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772824.233935:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772824.233947:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772824.233973:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534539008/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772824.233992:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534538112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772824.234046:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772824.234056:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772824.234064:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534538112:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772824.234081:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534538240:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772824.234127:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772824.234149:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772824/real 1790772824] req@ffff9c4efbcc1c00 x1877759534538240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772824.234174:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534538240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772824.234194:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772824.234201:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772824.234208:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534538240:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772824.234222:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534538368:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772824.234255:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772824.234273:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772824/real 1790772824] req@ffff9c4efbcc1880 x1877759534538368/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772824.234296:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534538368/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772824.234315:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772824.234322:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772824.234329:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534538368:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772824.234342:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534538496:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772824.234376:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772824.234395:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772824/real 1790772824] req@ffff9c4efbcc2300 x1877759534538496/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772824.234435:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534538496/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772824.234454:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772824.234461:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772824.234468:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534538496:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772824.234481:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534538624:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772824.234560:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772824.234598:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772824/real 1790772824] req@ffff9c4efbcc2680 x1877759534538624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772824.234639:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534538624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772824.234720:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772824.234728:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772824.234736:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534538624:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772824.234795:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534538752:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772824.234841:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772824.234865:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772824/real 1790772824] req@ffff9c4efbcc0700 x1877759534538752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772824.234892:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534538752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772824.234911:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772824.234918:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772824.234926:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534538752:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772824.234940:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534538880:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772824.234984:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772824.235009:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772824/real 1790772824] req@ffff9c4efbcc1f80 x1877759534538880/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772824.235080:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534538880/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772824.235113:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772824.235121:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772824.235128:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534538880:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772824.235140:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534539008:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772824.235182:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772824.235205:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772824/real 1790772824] req@ffff9c4efbcc0a80 x1877759534539008/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772824.235229:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534539008/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772824.235249:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772824.235257:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772824.235265:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534539008:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772824.296852:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772824.296854:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772824.296854:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772824.296855:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772824.360799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772824.360845:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772824.360989:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772824.488773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772824.488780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772824.552782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772824.616804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772824.680756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772824.680763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772824.680765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772824.744751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772824.808783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772824.808806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772824.808817:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772824.936726:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772824.936726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772824.936729:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772824.936814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772825.000789:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772825.000829:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772825.000837:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772825.000877:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772825.000886:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772825.000917:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772825.128786:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772825.192734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772825.256720:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772825.320756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772825.320792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772825.320825:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772825.320829:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772825.384762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772825.384866:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772825.384901:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772825.512807:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772825.512840:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772825.576768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772825.640749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772825.704798:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772825.704798:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772825.704848:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772825.768732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772825.832732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772825.832757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772825.832790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772825.960749:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772825.960749:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772825.960757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772825.960757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772826.024771:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772826.024771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772826.024776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772826.024786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772826.024803:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772826.024819:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772826.152765:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772826.216743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772826.280770:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772826.344741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772826.344751:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772826.344763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772826.344765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772826.408801:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772826.408878:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772826.408919:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772826.536753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772826.536762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772826.600733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772826.664775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772826.728767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772826.728784:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772826.728790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772826.792744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772826.856746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772826.856749:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772826.856795:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772826.984779:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772826.984787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772826.984788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772826.984796:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772827.048759:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772827.048786:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772827.048797:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772827.048801:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772827.048830:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772827.048848:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772827.176760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772827.240797:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772827.304748:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772827.368810:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772827.368819:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772827.368834:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772827.368843:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772827.432729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772827.432832:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772827.432921:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772827.560756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772827.560760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772827.624765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772827.688754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772827.752729:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772827.752741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772827.752764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772827.816793:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772827.880779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772827.880855:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772827.880862:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772828.008770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772828.008785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772828.008827:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772828.008870:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772828.072838:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772828.072839:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772828.072848:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772828.072882:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772828.072902:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772828.072917:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772828.200761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772828.264755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772828.328781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772828.392785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772828.392826:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772828.392828:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772828.392912:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772828.456803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772828.456889:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772828.456929:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772828.584777:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772828.584794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772828.648738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772828.712788:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772828.776720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772828.776724:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772828.776728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772828.840685:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772828.904714:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772828.904717:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772828.904740:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772829.032725:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772829.032727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772829.032729:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772829.032736:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772829.096715:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772829.096727:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772829.096730:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772829.096732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772829.096766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772829.096766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772829.224710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772829.288700:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772829.352703:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772829.352712:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772829.352718:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772829.352722:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772829.352731:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1825 00000100:00080000:2.0:1790772829.352740:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772829.352751:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772829.352784:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534539136/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772829.352807:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772829.352810:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772829.352812:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772829.352815:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1825 00000100:00080000:2.0:1790772829.352817:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772829.352821:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772829.352829:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772829.352838:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534539264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772829.352844:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534539136:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772829.352849:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772829.352855:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772829.352857:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772829.352861:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1825 00000100:00080000:2.0:1790772829.352863:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772829.352866:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772829.352877:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772829.352888:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534539392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772829.352895:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772829.352898:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772829.352899:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772829.352902:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1825 00000100:00080000:2.0:1790772829.352905:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772829.352908:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772829.352912:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772829.352917:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772829.352924:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534539520/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772829.352932:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772829.352935:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772829.352936:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772829.352939:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1825 00000100:00080000:2.0:1790772829.352942:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772829.352945:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772829.352948:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772829.352959:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534539648/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772829.352961:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772829/real 1790772829] req@ffff9c4efbcc3100 x1877759534539136/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772829.352972:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772829.352980:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772829.352983:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772829.352989:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1825 00000100:00080000:2.0:1790772829.352992:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772829.352996:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772829.352999:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772829.353002:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534539136/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772829.353009:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534539776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772829.353018:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772829.353019:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772829.353026:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772829.353031:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534539136:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772829.353032:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772829.353037:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772829.353042:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1825 00000100:00080000:2.0:1790772829.353044:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772829.353048:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772829.353052:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772829.353055:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534539264:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772829.353062:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534539904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772829.353072:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772829.353075:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772829.353077:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1790772829.353083:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772829.353084:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1825 00000100:00080000:2.0:1790772829.353090:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772829.353095:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772829.353100:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772829.353104:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772829/real 1790772829] req@ffff9c4efbcc0380 x1877759534539264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772829.353112:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534540032/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772829.353121:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534539264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772829.353123:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772829.353130:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772829.353133:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772829.353143:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1825 00000100:00080000:2.0:1790772829.353147:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772829.353151:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772829.353152:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790772829.353154:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:3.0:1790772829.353159:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772829.353163:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534539264:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772829.353166:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534540160/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772829.353173:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534539392:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772829.353198:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772829.353210:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772829/real 1790772829] req@ffff9c4efbcc0a80 x1877759534539392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772829.353224:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534539392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772829.353251:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772829.353257:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772829.353263:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534539392:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772829.353273:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534539520:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772829.353305:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772829.353320:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772829/real 1790772829] req@ffff9c4efbcc1f80 x1877759534539520/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772829.353337:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534539520/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772829.353350:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772829.353354:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772829.353359:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534539520:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772829.353367:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534539648:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772829.353427:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772829.353436:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772829/real 1790772829] req@ffff9c4efbcc0700 x1877759534539648/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772829.353444:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534539648/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772829.353451:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772829.353453:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772829.353455:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534539648:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772829.353459:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534539776:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772829.353473:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772829.353479:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772829/real 1790772829] req@ffff9c4efbcc2680 x1877759534539776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772829.353488:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534539776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772829.353499:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772829.353501:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772829.353503:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534539776:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772829.353519:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534539904:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772829.353531:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772829.353537:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772829/real 1790772829] req@ffff9c4efbcc2300 x1877759534539904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772829.353545:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534539904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772829.353551:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772829.353554:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772829.353556:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534539904:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772829.353562:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534540032:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772829.353590:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772829.353603:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772829/real 1790772829] req@ffff9c4efbcc1880 x1877759534540032/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772829.353616:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534540032/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772829.353629:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772829.353631:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772829.353634:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534540032:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772829.353638:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534540160:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772829.353674:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772829.353682:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772829/real 1790772829] req@ffff9c4efbcc1c00 x1877759534540160/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772829.353690:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534540160/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772829.353697:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772829.353700:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772829.353702:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534540160:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772829.416710:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772829.416713:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772829.416722:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772829.416724:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772829.480713:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772829.480740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772829.480755:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772829.608696:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772829.608707:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772829.672699:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772829.736726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772829.800767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772829.800769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772829.800817:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772829.864733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772829.928759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772829.928769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772829.928781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772830.056760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772830.056781:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772830.056785:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772830.056788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772830.120765:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772830.120782:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772830.120801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772830.120802:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772830.120839:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772830.120852:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772830.248766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772830.312757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772830.376779:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772830.440752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772830.440760:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772830.440775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772830.440793:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772830.504737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772830.504830:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772830.504886:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772830.632755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772830.632877:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772830.696744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772830.760769:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772830.824751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772830.824766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772830.824775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772830.888744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772830.952749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772830.952773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772830.952779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772831.080756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772831.080773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772831.080813:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772831.080816:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772831.144727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772831.144802:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772831.144817:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772831.144818:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772831.144818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772831.144849:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772831.272741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772831.336754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772831.400750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772831.464751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772831.464782:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772831.464808:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772831.464817:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772831.528783:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772831.528811:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772831.528963:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772831.656791:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772831.656875:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772831.720760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772831.784781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772831.848754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772831.848769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772831.848791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772831.912777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772831.976766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772831.976771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772831.976785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772832.104739:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772832.104805:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772832.104815:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772832.104816:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772832.168777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772832.168802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772832.168824:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772832.168836:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772832.168873:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772832.168880:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772832.296766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772832.360731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772832.424754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772832.488815:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772832.488829:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772832.488833:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772832.488840:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772832.552890:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772832.552939:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772832.553149:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772832.680736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772832.680767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772832.744759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772832.808803:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772832.872767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772832.872771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772832.872786:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772832.936778:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772833.000760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772833.000812:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772833.000827:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772833.128836:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772833.128837:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772833.128837:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772833.128842:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772833.192842:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772833.192851:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772833.192868:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772833.192905:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772833.192918:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772833.192924:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772833.320804:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772833.384772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772833.448741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772833.512781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772833.512830:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772833.512842:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772833.512852:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772833.576846:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772833.576850:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772833.576917:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772833.704778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772833.704834:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772833.768764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772833.832802:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772833.896749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772833.896753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772833.896765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772833.960733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772834.024749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772834.024756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772834.024794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772834.152764:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772834.152786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772834.152797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772834.152800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772834.216777:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772834.216793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772834.216818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772834.216832:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772834.216871:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772834.216881:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772834.344743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772834.408768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772834.472782:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772834.472797:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772834.472816:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772834.472827:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772834.472848:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1830 00000100:00080000:2.0:1790772834.472868:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772834.472904:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772834.472987:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534540288/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772834.473041:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772834.473047:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772834.473051:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772834.473057:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1830 00000100:00080000:2.0:1790772834.473063:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772834.473070:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772834.473086:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772834.473108:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534540416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772834.473136:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772834.473141:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534540288:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772834.473143:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772834.473152:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772834.473164:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1830 00000100:00080000:2.0:1790772834.473170:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772834.473176:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772834.473182:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772834.473208:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534540544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772834.473221:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772834.473226:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772834.473230:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772834.473237:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1830 00000100:00080000:2.0:1790772834.473263:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772834.473271:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772834.473279:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772834.473281:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772834.473352:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534540672/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772834.473368:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772834.473373:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772834.473377:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772834.473384:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1830 00000100:00080000:2.0:1790772834.473390:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772834.473393:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772834/real 1790772834] req@ffff9c4efbcc2a00 x1877759534540288/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772834.473404:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772834.473426:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772834.473443:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534540800/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772834.473462:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534540288/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772834.473463:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772834.473482:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772834.473489:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772834.473506:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1830 00000100:00080000:2.0:1790772834.473518:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772834.473532:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772834.473537:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790772834.473545:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1790772834.473554:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772834.473567:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534540288:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772834.473580:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534540928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772834.473621:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534540416:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772834.473624:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772834.473638:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772834.473642:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772834.473710:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1830 00000100:00080000:2.0:1790772834.473719:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772834.473746:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:3.0:1790772834.473757:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790772834.473762:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772834.473803:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772834/real 1790772834] req@ffff9c4efbcc1500 x1877759534540416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772834.473806:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534541056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772834.473932:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772834.473939:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772834.473945:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772834.473950:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534540416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772834.473957:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1830 00000100:00080000:2.0:1790772834.473970:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772834.473987:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772834.474005:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:3.0:1790772834.474027:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1790772834.474030:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534541184/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772834.474048:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772834.474070:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534540416:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772834.474073:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772834.474094:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772834.474098:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772834.474112:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1830 00000100:00080000:2.0:1790772834.474128:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772834.474131:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534540544:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772834.474137:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772834.474152:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772834.474170:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534541312/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772834.474205:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772834.474230:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772834/real 1790772834] req@ffff9c4efbcc1c00 x1877759534540544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772834.474246:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534540544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772834.474258:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772834.474263:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772834.474267:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534540544:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772834.474276:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534540672:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772834.474335:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772834.474350:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772834/real 1790772834] req@ffff9c4efbcc1880 x1877759534540672/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772834.474365:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534540672/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772834.474376:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772834.474381:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772834.474385:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534540672:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772834.474393:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534540800:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772834.474412:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772834.474423:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772834/real 1790772834] req@ffff9c4efbcc2300 x1877759534540800/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772834.474436:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534540800/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772834.474447:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772834.474450:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772834.474454:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534540800:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772834.474481:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534540928:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772834.474501:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772834.474512:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772834/real 1790772834] req@ffff9c4efbcc2680 x1877759534540928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772834.474534:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534540928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772834.474550:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772834.474554:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772834.474557:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534540928:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772834.474564:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534541056:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772834.474582:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772834.474592:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772834/real 1790772834] req@ffff9c4efbcc0700 x1877759534541056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772834.474605:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534541056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772834.474614:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772834.474618:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772834.474622:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534541056:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772834.474629:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534541184:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772834.474645:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772834.474686:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772834/real 1790772834] req@ffff9c4efbcc1f80 x1877759534541184/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772834.474700:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534541184/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772834.474720:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772834.474726:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772834.474732:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534541184:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772834.474762:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534541312:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772834.474802:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772834.474827:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772834/real 1790772834] req@ffff9c4efbcc0a80 x1877759534541312/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772834.474855:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534541312/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772834.474870:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772834.474875:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772834.474880:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534541312:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772834.536788:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772834.536788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772834.536842:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772834.536867:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772834.600805:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772834.600851:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772834.600859:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772834.728772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772834.728776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772834.792751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772834.856829:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772834.920760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772834.920773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772834.920773:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772834.984738:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772835.048751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772835.048826:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772835.048841:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772835.176788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772835.176805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772835.176811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772835.176819:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772835.240765:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772835.240777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772835.240816:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772835.240817:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772835.240854:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772835.240879:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772835.368757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772835.432748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772835.496806:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772835.560762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772835.560795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772835.560822:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772835.560829:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772835.624843:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772835.624887:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772835.624897:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772835.752794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772835.752824:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772835.816775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772835.880783:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772835.944747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772835.944755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772835.944804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772836.008750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772836.072783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772836.072866:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772836.072870:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772836.200759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772836.200776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772836.200845:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772836.200848:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772836.264779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772836.264801:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772836.264819:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772836.264854:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772836.264863:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772836.264886:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772836.392797:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772836.456768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772836.520780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772836.584801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772836.584810:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772836.584812:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772836.584902:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772836.648776:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772836.648868:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772836.648914:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772836.776767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772836.776791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772836.840763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772836.904846:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772836.968786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772836.968795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772836.968797:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772837.032780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772837.096837:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772837.096882:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772837.096892:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772837.224831:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772837.224896:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772837.224935:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772837.224936:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772837.288784:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772837.288812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772837.288813:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772837.288841:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772837.288870:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772837.288902:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772837.416761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772837.480772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772837.544746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772837.608761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772837.608794:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772837.608834:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772837.608898:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772837.672797:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772837.672913:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772837.672939:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772837.800784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772837.800792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772837.864791:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772837.928768:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772837.992794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772837.992805:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772837.992807:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772838.056777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772838.120793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772838.120821:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772838.120822:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772838.248795:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772838.248797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772838.248808:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772838.248814:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772838.312832:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772838.312840:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772838.312842:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772838.312845:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772838.312901:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772838.312926:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772838.440754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772838.504748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772838.568765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772838.632764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772838.632819:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772838.632825:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772838.632826:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772838.696762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772838.696872:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772838.696881:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772838.824753:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772838.824801:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772838.888763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772838.952794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772839.016751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772839.016752:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772839.016752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772839.080711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772839.144706:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772839.144713:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772839.144738:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772839.272765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772839.272843:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772839.272848:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772839.272853:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772839.336825:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772839.336825:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772839.336829:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772839.336844:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772839.336926:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772839.336967:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772839.464747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772839.528771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772839.592729:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772839.592738:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772839.592746:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772839.592752:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772839.592764:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1835 00000100:00080000:2.0:1790772839.592778:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772839.592804:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772839.592864:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534541440/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772839.592914:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772839.592918:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772839.592920:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772839.592925:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1835 00000100:00080000:2.0:1790772839.592930:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772839.592936:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772839.592949:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772839.592964:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534541568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772839.592975:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534541440:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772839.592980:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772839.592985:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772839.592986:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772839.592991:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1835 00000100:00080000:2.0:1790772839.592994:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772839.592998:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772839.593003:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772839.593014:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534541696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772839.593023:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772839.593037:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772839.593039:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772839.593042:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1835 00000100:00080000:2.0:1790772839.593056:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772839.593061:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772839.593066:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772839.593075:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772839.593079:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534541824/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772839.593107:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772839.593110:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772839.593112:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772839.593116:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1835 00000100:00080000:2.0:1790772839.593119:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772839.593122:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772839.593128:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772839.593141:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772839/real 1790772839] req@ffff9c4efbcc0380 x1877759534541440/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772839.593142:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534541952/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772839.593215:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772839.593223:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772839.593228:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772839.593228:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534541440/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772839.593239:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1835 00000100:00080000:2.0:1790772839.593247:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772839.593253:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772839.593256:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772839.593262:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790772839.593263:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772839.593270:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534541440:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772839.593281:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534542080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772839.593291:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772839.593295:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772839.593297:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772839.593305:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1835 00000100:00100000:3.0:1790772839.593305:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534541568:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772839.593314:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772839.593320:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772839.593333:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1790772839.593340:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772839.593354:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534542208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772839.593357:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772839/real 1790772839] req@ffff9c4efbcc3100 x1877759534541568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772839.593402:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772839.593409:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1790772839.593409:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534541568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772839.593414:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772839.593424:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1835 00000100:00080000:2.0:1790772839.593432:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772839.593441:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772839.593448:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:3.0:1790772839.593451:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772839.593460:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772839.593464:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534542336/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772839.593471:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534541568:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772839.593481:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772839.593486:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772839.593491:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772839.593491:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534541696:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772839.593501:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1835 00000100:00080000:2.0:1790772839.593505:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772839.593510:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772839.593515:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772839.593528:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534542464/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772839.593538:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772839.593559:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772839/real 1790772839] req@ffff9c4efbcc0a80 x1877759534541696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772839.593575:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534541696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772839.593583:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772839.593586:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772839.593589:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534541696:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772839.593595:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534541824:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772839.593613:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772839.593622:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772839/real 1790772839] req@ffff9c4efbcc1f80 x1877759534541824/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772839.593633:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534541824/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772839.593642:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772839.593644:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772839.593647:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534541824:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772839.593677:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534541952:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772839.593694:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772839.593704:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772839/real 1790772839] req@ffff9c4efbcc0700 x1877759534541952/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772839.593714:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534541952/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772839.593723:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772839.593726:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772839.593729:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534541952:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772839.593734:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534542080:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772839.593748:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772839.593756:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772839/real 1790772839] req@ffff9c4efbcc2680 x1877759534542080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772839.593766:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534542080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772839.593782:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772839.593785:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772839.593788:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534542080:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772839.593811:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534542208:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772839.593826:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772839.593840:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772839/real 1790772839] req@ffff9c4efbcc2300 x1877759534542208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772839.593856:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534542208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772839.593868:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772839.593883:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772839.593889:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534542208:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772839.593897:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534542336:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772839.593921:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772839.593931:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772839/real 1790772839] req@ffff9c4efbcc1880 x1877759534542336/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772839.593942:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534542336/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772839.593952:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772839.593954:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772839.593958:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534542336:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772839.593964:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534542464:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772839.593977:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772839.593985:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772839/real 1790772839] req@ffff9c4efbcc1c00 x1877759534542464/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772839.593994:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534542464/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772839.594013:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772839.594015:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772839.594018:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534542464:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772839.656734:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772839.656749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772839.656770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772839.656770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772839.720804:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772839.720822:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772839.720882:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772839.848760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772839.848765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772839.912816:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772839.976804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772840.040755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772840.040787:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772840.040789:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772840.104809:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772840.168770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772840.168786:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772840.168794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772840.296752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772840.296766:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772840.296801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772840.296842:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772840.360775:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772840.360786:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772840.360827:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772840.360879:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772840.360879:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772840.360883:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772840.488770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772840.552794:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772840.616714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772840.680762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772840.680773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772840.680812:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772840.680827:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772840.744828:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772840.744832:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772840.744840:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772840.872792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772840.872814:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772840.936749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772841.000831:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772841.064776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772841.064806:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772841.064806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772841.128736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772841.192708:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772841.192711:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772841.192766:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772841.320749:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772841.320756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772841.320789:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772841.320834:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772841.384754:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772841.384769:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772841.384811:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772841.384829:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772841.384831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772841.384857:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772841.512759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772841.576789:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772841.640872:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772841.705810:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772841.705879:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772841.706626:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772841.707623:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772841.768752:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772841.768848:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772841.768885:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772841.896742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772841.899331:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772841.960753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772842.024751:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772842.088766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772842.088787:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772842.088811:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772842.152751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772842.216771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772842.216812:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772842.216853:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772842.344760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772842.344760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772842.344830:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772842.346297:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772842.408764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772842.408765:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772842.408771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772842.408775:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772842.408820:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772842.408829:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772842.536716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772842.600709:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772842.664707:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772842.728750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772842.728773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772842.728788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772842.728788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772842.792743:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772842.792831:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772842.792916:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772842.920738:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772842.920787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772842.984755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772843.048779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772843.112709:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772843.112715:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772843.112795:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772843.176760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772843.240757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772843.240797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772843.240818:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772843.368765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772843.368765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772843.368778:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772843.368790:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772843.432747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772843.432756:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772843.432812:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772843.432836:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772843.432882:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772843.432915:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772843.560806:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772843.624771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772843.688781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772843.752734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772843.752784:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772843.752790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772843.752798:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772843.816799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772843.816886:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772843.816952:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772843.944756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772843.944789:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772844.008762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772844.072794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772844.136820:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772844.136864:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772844.136866:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772844.200783:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772844.264728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772844.264745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772844.264791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772844.392794:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772844.392808:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772844.392820:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772844.392821:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772844.456776:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772844.456782:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772844.456786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772844.456835:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772844.456850:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772844.456877:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772844.584733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772844.648731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772844.712796:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772844.712800:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772844.712815:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772844.712822:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772844.712841:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1840 00000100:00080000:2.0:1790772844.712858:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772844.712879:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772844.712944:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534542592/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772844.712985:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772844.712990:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772844.712993:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772844.713000:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1840 00000100:00080000:2.0:1790772844.713007:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772844.713034:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772844.713053:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772844.713075:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534542720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772844.713144:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534542592:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772844.713151:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772844.713167:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772844.713191:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772844.713201:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1840 00000100:00080000:2.0:1790772844.713207:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772844.713216:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772844.713225:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772844.713246:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534542848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772844.713259:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772844.713265:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772844.713268:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772844.713274:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1840 00000100:00080000:2.0:1790772844.713280:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772844.713287:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772844.713294:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772844.713309:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772844.713312:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534542976/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772844.713349:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772844.713356:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772844.713360:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772844.713368:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1840 00000100:00080000:2.0:1790772844.713376:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772844.713386:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772844.713397:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772844.713406:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772844/real 1790772844] req@ffff9c4efbcc1500 x1877759534542592/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772844.713417:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534543104/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772844.713451:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534542592/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772844.713455:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772844.713479:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772844.713489:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772844.713513:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1840 00000100:00080000:3.0:1790772844.713518:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772844.713522:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772844.713530:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772844.713538:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772844.713548:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534542592:192.168.204.150@tcp:250:kworker.0 00000008:00100000:2.0:1790772844.713558:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772844.713600:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534542720:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772844.713602:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534543232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772844.713639:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772844.713693:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772844.713701:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1790772844.713707:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772844.713718:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1840 00000100:00080000:2.0:1790772844.713734:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772844.713740:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772844/real 1790772844] req@ffff9c4efbcc2a00 x1877759534542720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772844.713757:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772844.713770:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534542720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790772844.713773:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772844.713805:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534543360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772844.713821:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772844.713851:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772844.713854:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772844.713861:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772844.713870:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772844.713880:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534542720:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772844.713885:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1840 00000100:00080000:2.0:1790772844.713901:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772844.713912:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772844.713921:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534542848:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790772844.713925:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772844.713948:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534543488/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772844.713965:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772844.713970:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772844.713973:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772844.713980:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1840 00000100:00080000:2.0:1790772844.713985:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1790772844.713989:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772844.713997:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772844.714005:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772844.714020:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772844/real 1790772844] req@ffff9c4efbcc1c00 x1877759534542848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772844.714024:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534543616/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772844.714064:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534542848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772844.714081:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772844.714122:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772844.714129:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534542848:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772844.714141:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534542976:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772844.714175:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772844.714191:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772844/real 1790772844] req@ffff9c4efbcc1880 x1877759534542976/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772844.714216:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534542976/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772844.714240:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772844.714244:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772844.714249:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534542976:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772844.714257:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534543104:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772844.714277:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772844.714288:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772844/real 1790772844] req@ffff9c4efbcc2300 x1877759534543104/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772844.714304:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534543104/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772844.714325:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772844.714329:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772844.714334:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534543104:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772844.714362:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534543232:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772844.714384:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772844.714397:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772844/real 1790772844] req@ffff9c4efbcc2680 x1877759534543232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772844.714412:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534543232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772844.714441:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772844.714445:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772844.714450:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534543232:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772844.714458:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534543360:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772844.714478:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772844.714490:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772844/real 1790772844] req@ffff9c4efbcc0700 x1877759534543360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772844.714505:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534543360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772844.714527:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772844.714531:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772844.714535:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534543360:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772844.714543:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534543488:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772844.714563:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772844.714574:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772844/real 1790772844] req@ffff9c4efbcc1f80 x1877759534543488/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772844.714589:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534543488/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772844.714602:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772844.714606:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772844.714611:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534543488:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772844.714619:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534543616:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772844.714638:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772844.714678:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772844/real 1790772844] req@ffff9c4efbcc0a80 x1877759534543616/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772844.714695:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534543616/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772844.714708:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772844.714712:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772844.714717:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534543616:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772844.776840:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772844.776864:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772844.776866:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772844.776867:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772844.840820:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772844.840840:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772844.840911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772844.968784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772844.968785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772845.032806:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772845.096788:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772845.160767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772845.160804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772845.160832:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772845.224756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772845.288790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772845.288849:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772845.288855:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772845.416767:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772845.416776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772845.416837:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772845.418460:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772845.480787:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772845.480820:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772845.480836:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772845.480870:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772845.480905:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772845.480905:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772845.608778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772845.672750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772845.736784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772845.800761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772845.800884:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772845.800885:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772845.800885:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772845.864768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772845.864808:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772845.865031:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772845.992807:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772845.992808:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772846.056777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772846.120802:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772846.184811:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772846.184874:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772846.184875:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772846.248765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772846.312791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772846.312810:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772846.314517:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772846.440734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772846.440782:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772846.440794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772846.440796:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772846.504754:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772846.504757:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772846.504779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772846.504791:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772846.504815:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772846.504825:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772846.632787:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772846.696813:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772846.760779:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772846.824769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772846.824770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772846.824829:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772846.824830:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772846.888866:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772846.888911:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772846.888914:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772847.016739:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772847.016759:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772847.080775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772847.144817:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772847.208770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772847.208774:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772847.208774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772847.272745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772847.336815:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772847.336821:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772847.336821:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772847.464773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772847.464839:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772847.464839:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772847.464842:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772847.528778:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772847.528787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772847.528838:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772847.528887:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772847.528892:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772847.528892:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772847.656762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772847.720750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772847.784760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772847.848780:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772847.848797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772847.848803:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772847.848809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772847.912787:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772847.912845:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772847.912978:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772848.040782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772848.040789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772848.104745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772848.168764:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772848.232845:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772848.232847:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772848.232847:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772848.296755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772848.360771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772848.360806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772848.360806:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772848.488756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772848.488767:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772848.488791:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772848.488841:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772848.552774:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772848.552784:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772848.552821:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772848.552833:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772848.552852:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772848.552858:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772848.680778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772848.744822:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772848.808761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772848.872753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772848.872764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772848.872811:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772848.872813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772848.936771:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772848.936852:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772848.936886:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772849.064770:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772849.064770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772849.128759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772849.192770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772849.256769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772849.256792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772849.256797:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772849.320765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772849.384773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772849.384776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772849.384781:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772849.512735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772849.512763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772849.512795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772849.512795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772849.576782:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772849.576803:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772849.576819:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772849.576854:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772849.576856:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772849.576879:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772849.704787:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772849.768748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790772849.832750:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772849.832752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772849.832763:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772849.832770:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772849.832785:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1845 00000100:00080000:2.0:1790772849.832802:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772849.832823:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772849.832950:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534543744/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772849.833012:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772849.833020:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772849.833023:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772849.833030:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1845 00000100:00080000:2.0:1790772849.833037:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772849.833047:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772849.833061:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772849.833084:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534543744:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772849.833091:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534543872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772849.833111:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772849.833116:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772849.833121:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772849.833126:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1845 00000100:00080000:2.0:1790772849.833131:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772849.833136:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772849.833143:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772849.833158:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534544000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772849.833171:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772849.833177:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772849.833179:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772849.833186:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1845 00000100:00080000:2.0:1790772849.833202:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772849.833209:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772849.833216:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:3.0:1790772849.833228:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772849.833239:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534544128/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772849.833253:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772849.833258:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772849.833261:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772849.833266:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1845 00000100:00080000:2.0:1790772849.833270:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772849.833275:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772849.833280:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772849.833296:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534544256/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772849.833309:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772849.833314:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772849.833317:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772849.833324:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1845 00000100:00080000:2.0:1790772849.833329:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772849.833339:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772849/real 1790772849] req@ffff9c4efbcc3100 x1877759534543744/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772849.833340:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772849.833352:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772849.833383:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534543744/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772849.833390:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534544384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772849.833416:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772849.833431:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772849.833435:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772849.833441:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534543744:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772849.833461:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772849.833467:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772849.833479:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1845 00000100:00080000:2.0:1790772849.833487:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772849.833498:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772849.833506:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534543872:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790772849.833511:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772849.833542:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534544512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772849.833558:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772849.833564:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772849.833567:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1790772849.833576:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772849.833583:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1845 00000100:00080000:2.0:1790772849.833593:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772849.833601:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772849.833609:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772849.833614:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772849/real 1790772849] req@ffff9c4efbcc0380 x1877759534543872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772849.833631:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534544640/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772849.833644:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534543872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772849.833704:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772849.833712:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772849.833716:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772849.833723:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772849.833727:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772849.833734:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534543872:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772849.833746:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1845 00000100:00100000:3.0:1790772849.833751:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534544000:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772849.833763:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772849.833778:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772849.833788:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:3.0:1790772849.833796:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772849.833817:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534544768/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772849.833820:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772849/real 1790772849] req@ffff9c4efbcc0a80 x1877759534544000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772849.833894:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534544000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772849.833908:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772849.833914:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772849.833919:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534544000:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772849.833927:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534544128:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772849.833952:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772849.833965:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772849/real 1790772849] req@ffff9c4efbcc1f80 x1877759534544128/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772849.833980:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534544128/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772849.834023:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772849.834029:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772849.834034:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534544128:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772849.834043:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534544256:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772849.834072:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772849.834087:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772849/real 1790772849] req@ffff9c4efbcc0700 x1877759534544256/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772849.834103:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534544256/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772849.834115:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772849.834119:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772849.834124:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534544256:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772849.834131:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534544384:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772849.834151:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772849.834165:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772849/real 1790772849] req@ffff9c4efbcc2680 x1877759534544384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772849.834189:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534544384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772849.834217:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772849.834225:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772849.834234:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534544384:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772849.834279:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534544512:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772849.834311:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772849.834327:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772849/real 1790772849] req@ffff9c4efbcc2300 x1877759534544512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772849.834343:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534544512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772849.834355:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772849.834360:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772849.834364:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534544512:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772849.834373:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534544640:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772849.834392:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772849.834404:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772849/real 1790772849] req@ffff9c4efbcc1880 x1877759534544640/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772849.834419:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534544640/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772849.834433:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772849.834437:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772849.834442:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534544640:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772849.834453:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534544768:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772849.834488:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772849.834512:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772849/real 1790772849] req@ffff9c4efbcc1c00 x1877759534544768/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772849.834540:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534544768/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772849.834561:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772849.834566:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772849.834571:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534544768:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772849.896787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772849.896802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772849.896802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772849.896804:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772849.960795:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772849.960814:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772849.960846:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772850.088756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772850.088761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772850.152742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772850.216779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772850.280750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772850.280793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772850.280824:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772850.344789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772850.408753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772850.408766:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772850.408808:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772850.536769:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772850.536769:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772850.536780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772850.536786:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772850.600749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772850.600791:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772850.600791:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772850.600800:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772850.600802:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772850.600833:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772850.728779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772850.792785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772850.856743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772850.920759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772850.920770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772850.920787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772850.920789:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772850.984710:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772850.984765:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772850.984769:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772851.112713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772851.112755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772851.176697:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772851.240715:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772851.304731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772851.304738:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772851.304741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772851.368744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772851.432773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772851.432775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772851.432801:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772851.560773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772851.560781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772851.560790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772851.560808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772851.624778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772851.624791:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772851.624802:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772851.624802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772851.624846:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772851.624847:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772851.752779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772851.816770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772851.880803:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772851.944772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772851.944778:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772851.944791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772851.944795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772852.008811:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772852.008862:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772852.008886:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772852.136813:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772852.136872:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772852.200787:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772852.264810:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772852.328758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772852.328764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772852.328786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772852.392745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772852.456756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772852.456762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772852.456815:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772852.584765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772852.584776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772852.584778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772852.584801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772852.648790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772852.648790:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772852.648801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772852.648831:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772852.648865:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772852.648873:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772852.776729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772852.840752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772852.904761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772852.968752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772852.968771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772852.968796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772852.968809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772853.032804:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772853.032826:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772853.032893:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772853.160754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772853.160786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772853.224780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772853.288796:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772853.352770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772853.352779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772853.352787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772853.416751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772853.480757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772853.480761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772853.480787:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772853.608783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772853.608803:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772853.608816:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772853.608824:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772853.672784:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772853.672820:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772853.672824:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772853.672826:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772853.672861:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772853.672885:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772853.800772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772853.864749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772853.928753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772853.992746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772853.992775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772853.992790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772853.992790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772854.056777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772854.056855:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772854.056973:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772854.184761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772854.184796:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772854.248772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772854.312803:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772854.376809:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772854.376809:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772854.376812:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772854.440748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772854.504763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772854.504763:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772854.504780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772854.632783:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772854.632783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772854.632812:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772854.632813:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772854.696740:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772854.696783:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772854.696803:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772854.696824:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772854.696824:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772854.696847:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772854.824759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772854.888775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772854.952783:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772854.952789:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772854.952800:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772854.952808:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772854.952824:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1850 00000100:00080000:2.0:1790772854.952842:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772854.952863:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772854.952940:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534544896/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772854.952977:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772854.952982:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772854.952985:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772854.952991:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1850 00000100:00080000:2.0:1790772854.952999:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772854.953007:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772854.953021:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772854.953037:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534544896:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772854.953047:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534545024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772854.953077:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772854.953082:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772854.953086:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772854.953092:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1850 00000100:00080000:2.0:1790772854.953097:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772854.953103:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772854.953115:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772854.953142:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534545152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772854.953164:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772854.953171:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772854.953176:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772854.953185:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1850 00000100:00080000:2.0:1790772854.953195:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772854.953208:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772854.953222:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534545024:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790772854.953223:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790772854.953259:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534545280/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772854.953281:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772854.953286:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772854.953290:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772854.953295:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534545152:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772854.953304:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1850 00000100:00080000:2.0:1790772854.953315:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772854.953325:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772854.953335:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772854.953346:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534545280:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772854.953357:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534545408/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772854.953376:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772854.953381:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772854.953386:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772854.953393:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1850 00000100:00100000:3.0:1790772854.953411:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534545408:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772854.953416:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772854.953433:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772854.953442:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772854.953464:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534545536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772854.953483:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772854.953489:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772854.953493:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772854.953497:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534545536:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772854.953504:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1850 00000100:00080000:2.0:1790772854.953515:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772854.953522:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772854.953528:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772854.953548:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534545664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772854.953564:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772854.953569:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772854.953573:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772854.953578:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1850 00000100:00080000:2.0:1790772854.953584:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772854.953592:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772854.953600:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772854.953602:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534545664:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772854.953626:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534545792/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772854.953643:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772854.953649:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772854.953700:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772854.953701:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534545792:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772854.953716:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1850 00000100:00080000:2.0:1790772854.953724:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772854.953745:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772854.953755:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772854.953775:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534545920/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772854.953883:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534545920:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772855.016742:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772855.016767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772855.016776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772855.016786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772855.080812:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772855.080844:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772855.080859:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772855.208782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772855.208800:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772855.272716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772855.336728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772855.400734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772855.400740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772855.400752:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772855.464787:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772855.528762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772855.528763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772855.528763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772855.656734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772855.656746:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772855.656757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772855.656771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772855.720784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772855.720785:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772855.720798:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772855.720812:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772855.720841:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772855.720862:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772855.848734:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772855.912778:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772855.976844:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772856.040815:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772856.040835:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772856.040836:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772856.040837:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772856.104786:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772856.104791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772856.104928:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772856.232776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772856.232776:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772856.296766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772856.360744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772856.424729:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772856.424729:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772856.424786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772856.488736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772856.552730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772856.552733:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772856.552733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772856.680793:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772856.680820:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772856.680826:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772856.680829:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772856.744770:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772856.744813:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772856.744829:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772856.744879:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772856.744931:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772856.744954:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772856.872749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772856.936767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772857.000870:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772857.064839:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772857.064842:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772857.064843:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772857.064862:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772857.128764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772857.128870:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772857.128929:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772857.256754:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772857.256780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772857.320742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772857.384866:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772857.448779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772857.448791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772857.448831:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772857.512776:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772857.576733:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772857.576748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772857.576797:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772857.704783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772857.704867:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772857.704897:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772857.704911:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772857.768786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772857.768885:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772857.768904:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772857.768904:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772857.768908:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772857.768966:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772857.896774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772857.960725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772858.025072:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:3.0:1790772858.025257:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:3.0:1790772858.025317:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1790772858.025343:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790772858.025428:0:2385:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:3.0:1790772858.025455:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790772858.025478:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790772858.025500:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790772858.025513:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772854/real 1790772858] req@ffff9c4efbcc0a80 x1877759534545920/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1790772858.025528:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790772858.025583:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534545920/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1790772858.025593:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790772858.025620:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:2.0:1790772858.025630:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000800:00000100:3.0:1790772858.025643:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:2.0:1790772858.025711:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772858.025725:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534545920:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772858.025749:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772854/real 1790772858] req@ffff9c4efbcc0700 x1877759534545664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772858.025787:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534545664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772858.025807:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772858.025817:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772858.025824:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534545664:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772858.025867:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772854/real 1790772858] req@ffff9c4efbcc2680 x1877759534545536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772858.025908:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534545536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772858.025943:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772858.025976:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772858.025985:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534545536:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772858.026003:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772854/real 1790772858] req@ffff9c4efbcc2300 x1877759534545408/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772858.026045:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534545408/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772858.026063:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772858.026070:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772858.026076:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534545408:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772858.026089:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772854/real 1790772858] req@ffff9c4efbcc1880 x1877759534545280/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772858.026112:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534545280/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772858.026133:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772858.026140:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772858.026146:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534545280:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772858.026158:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772854/real 1790772858] req@ffff9c4efbcc1500 x1877759534545024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772858.026209:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534545024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772858.026237:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772858.026244:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772858.026252:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534545024:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772858.026272:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772854/real 1790772858] req@ffff9c4efbcc1c00 x1877759534545152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772858.026298:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534545152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772858.026314:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772858.026321:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772858.026327:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534545152:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772858.026341:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772854/real 1790772858] req@ffff9c4efbcc2a00 x1877759534544896/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772858.026368:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534544896/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772858.026394:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772858.026402:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772858.026410:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534544896:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772858.026474:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772854/real 1790772858] req@ffff9c4efbcc1f80 x1877759534545792/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772858.026503:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534545792/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772858.026524:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772858.026531:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772858.026540:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534545792:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772858.088764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772858.088764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772858.088813:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772858.088813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772858.152876:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772858.152876:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772858.152940:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772858.280753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772858.280758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772858.344759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772858.408782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772858.472797:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772858.472815:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772858.472849:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772858.536805:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772858.600747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772858.600759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772858.600785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772858.728757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772858.728771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772858.728777:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772858.728791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772858.792751:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772858.792754:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772858.792777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772858.792803:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772858.792845:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772858.792846:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772858.920762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772858.984746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772859.048919:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772859.112747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772859.112749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772859.112756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772859.112756:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772859.176737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772859.176747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772859.176843:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772859.304760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772859.304767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772859.368762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772859.432798:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772859.496772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772859.496828:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772859.496856:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772859.560771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772859.624770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772859.624823:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772859.624823:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772859.752787:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772859.752842:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772859.752851:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772859.752853:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772859.816773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772859.816837:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772859.816861:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772859.816880:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772859.816882:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772859.816905:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772859.944764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772860.008733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772860.072913:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772860.072934:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772860.072947:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772860.072953:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772860.072982:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1855 00000100:00080000:2.0:1790772860.072997:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772860.073016:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772860.073073:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534546048/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772860.073120:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772860.073125:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772860.073128:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772860.073134:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1855 00000100:00080000:2.0:1790772860.073140:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772860.073165:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772860.073180:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772860.073203:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534546176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772860.073217:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534546048:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772860.073238:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772860.073264:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772860.073268:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772860.073278:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1855 00000100:00080000:2.0:1790772860.073287:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772860.073297:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772860.073308:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772860.073333:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534546304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1790772860.073341:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772860.073375:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772860.073382:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772860.073387:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772860.073398:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1855 00000100:00080000:2.0:1790772860.073407:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772860.073419:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772860/real 1790772860] req@ffff9c4efbcc1f80 x1877759534546048/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772860.073432:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772860.073450:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772860.073472:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534546048/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772860.073481:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534546432/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772860.073519:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772860.073526:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772860.073530:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772860.073539:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1855 00000100:00080000:2.0:1790772860.073548:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772860.073559:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:3.0:1790772860.073565:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772860.073580:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790772860.073581:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772860.073593:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534546048:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772860.073622:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534546560/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772860.073645:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772860.073646:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534546176:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772860.073712:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772860.073716:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772860.073730:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1855 00000100:00080000:2.0:1790772860.073741:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772860.073755:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772860.073766:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1790772860.073809:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772860.073820:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534546688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772860.073843:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772860.073847:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772860/real 1790772860] req@ffff9c4efbcc2a00 x1877759534546176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772860.073854:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772860.073862:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772860.073881:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1855 00000100:00080000:2.0:1790772860.073893:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772860.073901:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534546176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772860.073911:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772860.073929:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1790772860.073950:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1790772860.073959:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534546816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772860.073967:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772860.073986:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534546176:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772860.073995:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772860.074004:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1790772860.074007:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534546304:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772860.074010:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772860.074029:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1855 00000100:00080000:2.0:1790772860.074039:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772860.074051:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772860.074062:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:3.0:1790772860.074073:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772860.074092:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534546944/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772860.074108:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772860/real 1790772860] req@ffff9c4efbcc1c00 x1877759534546304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772860.074163:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772860.074171:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772860.074175:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772860.074184:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1855 00000100:00080000:2.0:1790772860.074191:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772860.074200:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772860.074209:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772860.074230:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534547072/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772860.074504:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534546304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772860.074525:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772860.074533:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772860.074540:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534546304:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772860.074553:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534546432:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772860.074602:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772860.074628:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772860/real 1790772860] req@ffff9c4efbcc1500 x1877759534546432/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772860.074677:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534546432/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772860.074698:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772860.074705:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772860.074712:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534546432:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772860.074748:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534546560:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772860.074813:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772860.074840:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772860/real 1790772860] req@ffff9c4efbcc1880 x1877759534546560/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772860.074865:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534546560/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772860.074883:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772860.074891:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772860.074898:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534546560:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772860.074912:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534546688:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772860.074953:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772860.074973:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772860/real 1790772860] req@ffff9c4efbcc2300 x1877759534546688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772860.074995:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534546688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772860.075026:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772860.075033:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772860.075040:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534546688:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772860.075053:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534546816:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772860.075092:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772860.075113:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772860/real 1790772860] req@ffff9c4efbcc2680 x1877759534546816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772860.075138:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534546816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772860.075156:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772860.075163:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772860.075171:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534546816:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772860.075184:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534546944:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772860.075222:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772860.075243:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772860/real 1790772860] req@ffff9c4efbcc0700 x1877759534546944/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772860.075267:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534546944/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772860.075286:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772860.075293:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772860.075301:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534546944:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772860.075314:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534547072:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772860.075356:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772860.075378:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772860/real 1790772860] req@ffff9c4efbcc0a80 x1877759534547072/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772860.075403:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534547072/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772860.075422:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772860.075429:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772860.075435:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534547072:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772860.136752:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772860.136784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772860.136816:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772860.136818:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772860.200779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772860.200878:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772860.200989:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772860.328766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772860.328779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772860.392761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772860.456805:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772860.520752:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772860.520813:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772860.520813:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772860.584733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772860.648875:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772860.648877:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772860.648877:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772860.776843:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772860.776845:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772860.776858:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772860.776889:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772860.840787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772860.840798:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772860.840799:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772860.840849:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772860.840880:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772860.840884:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772860.968824:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772861.032812:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772861.096930:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772861.160848:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772861.160850:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772861.160928:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772861.162595:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772861.224846:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772861.224984:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772861.225006:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772861.352809:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772861.352850:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772861.416817:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772861.480825:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772861.544845:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772861.544845:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772861.544853:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772861.608808:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772861.672831:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772861.672840:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772861.672875:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772861.800837:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772861.800850:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772861.800869:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772861.800872:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772861.864822:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772861.864826:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772861.864869:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772861.864904:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772861.864922:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772861.864960:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772861.992835:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772862.056740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772862.120819:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772862.184738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772862.184741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772862.184743:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772862.184780:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772862.248742:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772862.248803:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772862.248994:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772862.376789:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772862.376822:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772862.440811:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772862.504792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772862.568782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772862.568786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772862.568813:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772862.632918:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772862.696783:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772862.696798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772862.696832:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772862.824806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772862.824813:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772862.824825:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772862.824839:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772862.888795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772862.888834:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772862.888834:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772862.888844:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772862.888874:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772862.888891:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772863.016765:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772863.080777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772863.144762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772863.208785:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772863.208785:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772863.208826:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772863.208843:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772863.272812:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772863.272886:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772863.272924:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772863.400770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772863.400771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772863.464794:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772863.528826:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772863.592788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772863.592795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772863.592813:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772863.656860:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772863.720837:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772863.720887:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772863.720903:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772863.848800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772863.848800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772863.848808:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772863.848839:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772863.912815:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772863.912821:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772863.912825:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772863.912837:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772863.912900:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772863.912911:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772864.040774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772864.104799:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772864.168758:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772864.232757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772864.232798:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772864.232829:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772864.232858:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772864.296779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772864.296862:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772864.296999:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772864.424811:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772864.424829:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772864.488836:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772864.552817:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772864.616765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772864.616795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772864.616837:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772864.680852:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772864.744805:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772864.744808:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772864.744855:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772864.872795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772864.872795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772864.872814:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772864.872815:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772864.936826:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772864.936843:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772864.936843:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772864.936856:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772864.936892:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772864.936976:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772865.064728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772865.128734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790772865.192817:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772865.192825:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772865.192829:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790772865.192831:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772865.192838:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1861 00000100:00080000:0.0:1790772865.192848:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772865.192858:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772865.192892:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534547200/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772865.192914:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772865.192917:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772865.192919:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772865.192922:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1861 00000100:00080000:0.0:1790772865.192924:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772865.192928:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772865.192936:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772865.192947:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534547328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772865.192953:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534547200:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772865.192961:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772865.192965:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772865.192966:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772865.192969:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1861 00000100:00080000:0.0:1790772865.192971:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772865.192976:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772865.192979:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772865.192987:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534547456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772865.192994:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772865.192997:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772865.192998:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772865.193001:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1861 00000100:00080000:0.0:1790772865.193003:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772865.193007:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772865.193010:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772865.193018:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534547584/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772865.193025:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772865.193028:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772865.193030:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772865.193032:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1861 00000100:00080000:0.0:1790772865.193036:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1790772865.193040:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772865.193044:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772865.193048:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772865.193057:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534547712/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772865.193063:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772865.193065:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772865.193066:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772865.193069:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1861 00000100:00080000:0.0:1790772865.193073:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772865.193077:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772865.193081:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772865/real 1790772865] req@ffff9c4ec7a7df80 x1877759534547200/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790772865.193083:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772865.193096:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534547840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772865.193104:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534547200/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772865.193106:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772865.193112:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772865.193116:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772865.193128:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1861 00000100:00080000:3.0:1790772865.193128:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772865.193136:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772865.193136:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772865.193142:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534547200:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772865.193148:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772865.193155:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772865.193181:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534547328:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772865.193187:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534547968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772865.193201:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1790772865.193203:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772865.193207:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772865.193210:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772865.193215:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1861 00000100:00100000:3.0:1790772865.193218:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772865/real 1790772865] req@ffff9c4ec7a7f480 x1877759534547328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772865.193221:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772865.193230:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772865.193234:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534547328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790772865.193235:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772865.193252:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534548096/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772865.193254:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772865.193263:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772865.193270:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534547328:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772865.193279:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790772865.193281:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534547456:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772865.193287:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772865.193290:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772865.193296:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1861 00000800:00000100:3.0:1790772865.193301:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790772865.193303:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772865.193312:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772865/real 1790772865] req@ffff9c4ec7a7c380 x1877759534547456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772865.193313:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772865.193322:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772865.193333:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534547456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772865.193338:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534548224/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772865.193360:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772865.193367:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772865.193372:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534547456:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772865.193378:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534547584:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772865.193396:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772865.193413:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772865/real 1790772865] req@ffff9c4ec7a7d880 x1877759534547584/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772865.193422:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534547584/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772865.193429:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772865.193432:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772865.193434:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534547584:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772865.193438:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534547712:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772865.193450:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772865.193457:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772865/real 1790772865] req@ffff9c4ec7a7ed80 x1877759534547712/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772865.193465:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534547712/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772865.193471:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772865.193473:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772865.193475:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534547712:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772865.193479:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534547840:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772865.193496:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772865.193503:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772865/real 1790772865] req@ffff9c4ec7a7ca80 x1877759534547840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772865.193511:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534547840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772865.193523:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772865.193525:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772865.193527:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534547840:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772865.193540:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534547968:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772865.193552:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772865.193558:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772865/real 1790772865] req@ffff9c4ec7a7e300 x1877759534547968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772865.193566:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534547968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772865.193572:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772865.193574:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772865.193577:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534547968:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772865.193580:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534548096:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772865.193589:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772865.193595:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772865/real 1790772865] req@ffff9c4ec7a7fb80 x1877759534548096/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772865.193603:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534548096/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772865.193614:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772865.193616:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772865.193619:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534548096:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772865.193622:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534548224:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772865.193631:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772865.193637:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772865/real 1790772865] req@ffff9c4ec7a7c700 x1877759534548224/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772865.193646:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534548224/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772865.193678:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772865.193682:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772865.193685:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534548224:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772865.256769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772865.256770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772865.256777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772865.256781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772865.320789:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772865.320814:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772865.320832:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772865.448739:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772865.448746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772865.512746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772865.576774:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772865.640787:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772865.640801:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772865.640802:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772865.704821:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772865.768788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772865.768816:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772865.768853:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772865.896801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772865.896827:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772865.896843:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772865.896855:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772865.960787:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772865.960830:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772865.960844:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772865.960875:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772865.960887:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772865.960901:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772866.088816:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772866.152774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772866.216786:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772866.280808:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772866.280829:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772866.280841:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772866.280842:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772866.344836:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772866.345048:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772866.346832:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772866.472773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772866.472785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772866.536782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772866.600759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772866.664771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772866.664786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772866.664793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772866.728794:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772866.792796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772866.792814:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772866.792816:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772866.920750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772866.920773:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772866.920797:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772866.920820:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772866.984763:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772866.984787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772866.984807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772866.984856:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772866.984860:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772866.984863:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772867.112794:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772867.176748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772867.240746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772867.304792:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772867.304792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772867.304847:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772867.304878:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772867.368843:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772867.368844:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772867.368881:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772867.496776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772867.496778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772867.560753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772867.624814:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772867.688759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772867.688810:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772867.688830:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772867.752800:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772867.816775:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772867.816778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772867.816778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772867.944847:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772867.944847:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772867.944850:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772867.944850:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772868.008806:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772868.008813:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772868.008814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772868.008833:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772868.008884:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772868.008894:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772868.136774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772868.200761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772868.264761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772868.328791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772868.328798:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772868.328846:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772868.328976:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772868.392755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772868.392829:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772868.392930:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772868.520790:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772868.520813:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772868.584779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772868.648764:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772868.712850:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772868.712863:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772868.712864:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772868.776796:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772868.840777:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772868.840787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772868.840790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772868.968783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772868.968784:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772868.968811:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772868.968811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772869.032771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772869.032793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772869.032793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772869.032812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772869.032833:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772869.032863:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772869.160744:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772869.224756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772869.288746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772869.352763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772869.352796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772869.352804:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772869.352812:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772869.416819:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772869.416842:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772869.416906:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772869.544748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772869.544792:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772869.608746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772869.672747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772869.736741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772869.736756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772869.736798:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772869.800768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772869.864803:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772869.864811:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772869.864841:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772869.992766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772869.992791:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772869.992844:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772869.992845:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772870.056757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772870.056771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772870.056809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772870.056835:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772870.056932:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772870.056955:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772870.184781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772870.248776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772870.312755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790772870.312795:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772870.312806:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790772870.312813:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772870.312828:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1866 00000100:00080000:0.0:1790772870.312856:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772870.312875:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772870.312932:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534548352/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772870.312981:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772870.312987:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772870.312990:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772870.312995:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1866 00000100:00080000:0.0:1790772870.313000:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772870.313006:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772870.313021:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772870.313042:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534548480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772870.313055:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772870.313061:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772870.313063:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772870.313070:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1866 00000100:00080000:0.0:1790772870.313074:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772870.313079:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772870.313086:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772870.313092:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534548352:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772870.313104:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534548608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772870.313121:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772870.313126:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772870.313129:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772870.313134:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1866 00000100:00080000:0.0:1790772870.313139:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772870.313145:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772870.313151:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772870.313168:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534548736/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772870.313181:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772870.313186:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772870.313189:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772870.313194:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1866 00000100:00080000:0.0:1790772870.313200:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772870.313210:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772870.313216:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:3.0:1790772870.313228:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772870.313235:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534548864/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772870.313250:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772870.313256:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772870.313260:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772870.313265:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1866 00000100:00080000:0.0:1790772870.313270:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772870.313275:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772870.313282:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772870.313299:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534548992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772870.313306:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772870/real 1790772870] req@ffff9c4ec7a7d180 x1877759534548352/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772870.313371:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772870.313384:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790772870.313393:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772870.313401:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1866 00000100:00080000:0.0:1790772870.313407:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772870.313412:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534548352/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772870.313419:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772870.313431:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1790772870.313449:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790772870.313456:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534549120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772870.313463:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772870.313480:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534548352:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790772870.313486:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772870.313496:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772870.313500:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790772870.313503:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534548480:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790772870.313511:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1866 00000100:00080000:0.0:1790772870.313517:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772870.313524:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772870.313532:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:3.0:1790772870.313551:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772870.313562:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534549248/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1790772870.313574:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772870/real 1790772870] req@ffff9c4ec7a7c700 x1877759534548480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772870.313640:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772870.313689:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772870.313693:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772870.313702:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1866 00000100:00080000:0.0:1790772870.313711:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790772870.313721:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534548480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772870.313724:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790772870.313738:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772870.313765:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534549376/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772870.313777:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772870.313789:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772870.313796:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534548480:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772870.313843:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534548608:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772870.313880:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772870.313897:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772870/real 1790772870] req@ffff9c4ec7a7fb80 x1877759534548608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772870.313915:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534548608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772870.313935:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772870.313940:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772870.313945:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534548608:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772870.313953:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534548736:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772870.313975:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772870.313987:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772870/real 1790772870] req@ffff9c4ec7a7e300 x1877759534548736/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772870.314002:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534548736/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772870.314015:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772870.314020:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772870.314024:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534548736:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772870.314031:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534548864:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772870.314057:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772870.314078:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772870/real 1790772870] req@ffff9c4ec7a7ca80 x1877759534548864/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772870.314103:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534548864/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772870.314122:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772870.314130:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772870.314139:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534548864:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772870.314152:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534548992:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772870.314190:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772870.314207:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772870/real 1790772870] req@ffff9c4ec7a7ed80 x1877759534548992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772870.314223:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534548992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772870.314244:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772870.314249:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772870.314254:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534548992:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772870.314262:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534549120:192.168.204.150@tcp:38:kworker.0 00000800:00000100:3.0:1790772870.314283:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772870.314295:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772870/real 1790772870] req@ffff9c4ec7a7d880 x1877759534549120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772870.314310:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534549120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772870.314321:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772870.314325:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772870.314329:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534549120:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772870.314336:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534549248:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772870.314356:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772870.314368:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772870/real 1790772870] req@ffff9c4ec7a7c380 x1877759534549248/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772870.314383:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534549248/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772870.314396:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772870.314400:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772870.314404:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534549248:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772870.314411:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534549376:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790772870.314430:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772870.314441:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772870/real 1790772870] req@ffff9c4ec7a7f480 x1877759534549376/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772870.314456:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534549376/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772870.314468:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772870.314472:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772870.314476:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534549376:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772870.376766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772870.376772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772870.376815:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772870.376815:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772870.440783:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772870.440797:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772870.440816:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772870.568768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772870.568773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772870.632745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772870.696816:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772870.760765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772870.760765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772870.760770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772870.824755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772870.888762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772870.888808:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772870.888811:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772871.016776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772871.016827:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772871.016865:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772871.016875:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772871.080785:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772871.080793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772871.080794:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772871.080838:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772871.080880:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772871.080901:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772871.208743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772871.272741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772871.336757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772871.400769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772871.400796:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772871.400797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772871.400804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772871.464779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772871.464781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772871.464847:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772871.592759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772871.592762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772871.656702:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772871.720804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772871.784733:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772871.784747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772871.784752:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772871.848767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772871.912722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772871.912739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772871.912772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772872.040735:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772872.040744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772872.040750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772872.040751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772872.104791:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772872.104794:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772872.104794:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772872.104823:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772872.104877:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772872.104894:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772872.232731:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772872.296757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772872.360769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772872.424708:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772872.424713:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772872.424725:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772872.424732:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772872.488705:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772872.488747:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772872.488771:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772872.616748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772872.616762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772872.680708:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772872.744828:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772872.808748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772872.808748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772872.808791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772872.872736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772872.936749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772872.936756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772872.936789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772873.064785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772873.064843:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772873.064857:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772873.064897:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772873.128767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772873.128808:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772873.128814:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772873.128824:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772873.128842:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772873.128855:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772873.256800:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772873.320763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772873.384739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772873.448742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772873.448754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772873.448763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772873.448781:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772873.512800:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772873.512803:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772873.512894:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772873.640729:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772873.640744:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772873.704723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772873.768717:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772873.832725:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772873.832727:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772873.832731:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772873.896723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772873.960728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772873.960728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772873.960731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772874.088736:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772874.088737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772874.088737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772874.088738:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772874.152718:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772874.152729:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772874.152741:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772874.152746:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772874.152763:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772874.152768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772874.280716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772874.344750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772874.408716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772874.472712:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772874.472727:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772874.472729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772874.472752:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772874.536733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772874.536766:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772874.536885:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772874.664762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772874.664763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772874.728717:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772874.792773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772874.856698:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772874.856719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772874.856719:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772874.920734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790772874.984779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772874.984779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772874.984785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772875.112739:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772875.112739:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772875.112740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772875.112754:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772875.176767:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772875.176767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772875.176803:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772875.176804:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772875.176834:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772875.176855:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772875.304739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772875.368729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790772875.432717:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772875.432725:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1790772875.432726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772875.432729:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772875.432739:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1871 00000100:00080000:1.0:1790772875.432748:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772875.432758:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772875.432789:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534549504/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772875.432818:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772875.432821:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772875.432823:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772875.432826:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1871 00000100:00080000:1.0:1790772875.432828:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772875.432832:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772875.432839:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772875.432850:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534549632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772875.432858:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772875.432861:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1790772875.432861:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534549504:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772875.432866:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772875.432873:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1871 00000100:00080000:1.0:1790772875.432875:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772875.432885:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772875.432893:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772875.432904:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534549760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772875.432912:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772875.432915:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772875.432916:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772875.432919:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1871 00000100:00080000:1.0:1790772875.432923:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772875.432928:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772875.432931:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772875.432937:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772875.432953:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534549888/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772875.432965:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772875.432969:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772875.432971:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772875.432977:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1871 00000100:00100000:2.0:1790772875.432981:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772875/real 1790772875] req@ffff9c4efac4df80 x1877759534549504/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772875.432982:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772875.432989:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772875.432996:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772875.433009:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534550016/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772875.433013:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534549504/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772875.433031:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772875.433034:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772875.433036:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772875.433040:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772875.433044:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772875.433047:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1871 00000100:00100000:2.0:1790772875.433052:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534549504:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772875.433053:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772875.433061:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772875.433065:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772875.433080:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534549632:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772875.433083:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534550144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772875.433098:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772875.433101:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772875.433103:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790772875.433106:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772875.433110:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1871 00000100:00080000:1.0:1790772875.433116:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772875.433121:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772875/real 1790772875] req@ffff9c4efac4c700 x1877759534549632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772875.433122:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772875.433130:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772875.433141:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534549632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772875.433152:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534550272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772875.433162:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772875.433164:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772875.433165:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772875.433170:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772875.433173:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772875.433178:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1871 00000100:00100000:2.0:1790772875.433182:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534549632:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772875.433183:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772875.433191:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772875.433193:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534549760:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790772875.433197:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790772875.433210:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534550400/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772875.433217:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772875.433227:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772875.433229:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772875/real 1790772875] req@ffff9c4efac4ed80 x1877759534549760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772875.433234:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772875.433239:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772875.433245:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534549760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772875.433249:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1871 00000100:00080000:1.0:1790772875.433259:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772875.433260:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772875.433266:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772875.433270:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772875.433272:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534549760:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790772875.433276:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772875.433280:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534549888:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772875.433291:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534550528/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772875.433306:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772875.433322:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772875/real 1790772875] req@ffff9c4efac4dc00 x1877759534549888/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772875.433336:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534549888/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772875.433347:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772875.433350:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772875.433353:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534549888:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772875.433358:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534550016:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772875.433374:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772875.433382:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772875/real 1790772875] req@ffff9c4efac4f800 x1877759534550016/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772875.433390:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534550016/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772875.433397:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772875.433399:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772875.433402:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534550016:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772875.433415:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534550144:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772875.433426:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772875.433433:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772875/real 1790772875] req@ffff9c4efac4f480 x1877759534550144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772875.433441:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534550144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772875.433452:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772875.433454:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772875.433456:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534550144:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772875.433460:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534550272:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772875.433470:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772875.433477:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772875/real 1790772875] req@ffff9c4efac4e680 x1877759534550272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772875.433485:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534550272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772875.433507:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772875.433509:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772875.433512:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534550272:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772875.433516:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534550400:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772875.433531:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772875.433539:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772875/real 1790772875] req@ffff9c4efac4ea00 x1877759534550400/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772875.433548:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534550400/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772875.433555:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772875.433557:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772875.433560:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534550400:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772875.433564:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534550528:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772875.433574:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772875.433580:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772875/real 1790772875] req@ffff9c4efac4e300 x1877759534550528/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772875.433587:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534550528/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772875.433594:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772875.433597:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772875.433599:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534550528:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772875.496746:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772875.496747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772875.496747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772875.496747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772875.560733:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772875.560784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772875.560793:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772875.688712:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772875.688727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772875.752725:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772875.816728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772875.880709:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772875.880709:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772875.880725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772875.944732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772876.008719:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772876.008720:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772876.008735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772876.136718:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772876.136719:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772876.136723:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772876.136739:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772876.200725:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772876.200731:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772876.200739:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772876.200753:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772876.200774:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772876.200782:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772876.328715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772876.392755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772876.456737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772876.520728:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772876.520730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772876.520730:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772876.520785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772876.584750:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772876.584762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772876.584918:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772876.712728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772876.712728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772876.776726:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772876.840787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772876.904700:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772876.904715:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772876.904719:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772876.968725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772877.032771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772877.032777:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772877.032778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772877.160744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772877.160744:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772877.160762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772877.160771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772877.224702:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772877.224734:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772877.224753:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772877.224776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772877.224776:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772877.224776:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772877.352709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772877.416744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772877.480720:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772877.544773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772877.544774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772877.544774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772877.544784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772877.608747:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772877.608765:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772877.608842:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772877.736710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772877.736741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772877.800768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772877.864733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772877.928705:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772877.928705:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772877.928735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772877.992716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772878.056731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772878.056790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772878.056803:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772878.184725:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772878.184795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772878.184795:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772878.184796:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772878.248722:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772878.248748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772878.248773:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772878.248789:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772878.248791:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772878.248797:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772878.376705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772878.440734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772878.504721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772878.568741:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772878.568741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772878.568741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772878.568768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772878.632770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772878.632799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772878.632878:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772878.760719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772878.760723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772878.824738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772878.888741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772878.952703:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772878.952703:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772878.952732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772879.016737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772879.080730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772879.080730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772879.080733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772879.208732:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772879.208739:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772879.208748:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772879.208754:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772879.272719:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772879.272752:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772879.272757:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772879.272757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772879.272766:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772879.272787:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772879.400757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772879.464716:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772879.528769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772879.592739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772879.592740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772879.592756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772879.592764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772879.656710:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772879.656730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772879.656804:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772879.784721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772879.784722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772879.848712:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772879.912726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772879.976711:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772879.976713:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772879.976719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772880.040728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772880.104738:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772880.104738:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772880.104739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772880.232721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772880.232721:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772880.232728:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772880.232729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772880.296715:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772880.296715:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772880.296722:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772880.296730:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772880.296751:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772880.296767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772880.424704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772880.488738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772880.552709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772880.552755:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772880.552761:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772880.552765:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772880.552773:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1876 00000100:00080000:3.0:1790772880.552782:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772880.552791:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772880.552824:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534550656/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772880.552848:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772880.552850:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772880.552852:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772880.552856:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1876 00000100:00080000:3.0:1790772880.552858:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772880.552861:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772880.552869:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772880.552881:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534550784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772880.552889:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772880.552892:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772880.552895:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772880.552897:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534550656:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772880.552900:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1876 00000100:00080000:3.0:1790772880.552905:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772880.552909:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772880.552913:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772880.552920:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534550912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772880.552926:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772880.552929:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772880.552930:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772880.552932:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1876 00000100:00080000:3.0:1790772880.552934:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772880.552937:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772880.552939:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772880.552945:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534551040/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772880.552950:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772880.552952:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772880.552953:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772880.552964:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1876 00000100:00080000:3.0:1790772880.552967:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772880.552970:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772880.552974:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772880.552983:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534551168/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772880.552989:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772880.552992:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772880.552993:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772880.552996:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1876 00000100:00080000:3.0:1790772880.552998:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772880.553001:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772880.553005:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790772880.553007:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772880.553016:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534551296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772880.553023:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772880.553026:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772880.553028:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772880.553031:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1876 00000100:00080000:3.0:1790772880.553033:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772880.553036:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772880.553040:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772880.553050:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772880/real 1790772880] req@ffff9c4ef73c0e00 x1877759534550656/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772880.553051:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534551424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772880.553084:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772880.553088:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772880.553089:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772880.553093:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534550656/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772880.553096:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1876 00000100:00080000:3.0:1790772880.553101:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772880.553110:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772880.553111:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790772880.553113:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:2.0:1790772880.553117:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772880.553122:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534550656:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772880.553126:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534551552/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772880.553134:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772880.553136:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772880.553138:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772880.553141:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1876 00000100:00080000:3.0:1790772880.553143:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772880.553149:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534550784:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772880.553149:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772880.553155:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772880.553167:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534551680/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772880.553174:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772880.553188:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772880/real 1790772880] req@ffff9c4ef73c1c00 x1877759534550784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772880.553202:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534550784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772880.553219:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772880.553223:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772880.553228:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534550784:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772880.553235:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534550912:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772880.553257:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772880.553272:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772880/real 1790772880] req@ffff9c4ef73c0000 x1877759534550912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772880.553286:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534550912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772880.553295:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772880.553298:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772880.553301:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534550912:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772880.553306:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534551040:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772880.553323:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772880.553331:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772880/real 1790772880] req@ffff9c4ef73c2300 x1877759534551040/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772880.553340:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534551040/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772880.553347:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772880.553350:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772880.553352:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534551040:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772880.553356:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534551168:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772880.553367:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772880.553374:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772880/real 1790772880] req@ffff9c4ef73c0700 x1877759534551168/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772880.553382:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534551168/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772880.553404:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772880.553407:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772880.553409:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534551168:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772880.553414:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534551296:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772880.553429:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772880.553437:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772880/real 1790772880] req@ffff9c4ef73c2a00 x1877759534551296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772880.553446:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534551296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772880.553456:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772880.553459:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772880.553461:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534551296:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772880.553466:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534551424:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772880.553475:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772880.553481:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772880/real 1790772880] req@ffff9c4ef73c3800 x1877759534551424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772880.553489:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534551424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772880.553495:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772880.553498:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772880.553500:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534551424:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772880.553504:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534551552:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772880.553513:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772880.553519:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772880/real 1790772880] req@ffff9c4ef73c0a80 x1877759534551552/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772880.553526:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534551552/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772880.553533:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772880.553535:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772880.553538:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534551552:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772880.553556:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534551680:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772880.553566:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772880.553573:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772880/real 1790772880] req@ffff9c4ef73c1500 x1877759534551680/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772880.553589:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534551680/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772880.553596:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772880.553598:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772880.553601:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534551680:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772880.616725:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772880.616726:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772880.616729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772880.616732:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772880.680735:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772880.680735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772880.680835:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772880.808712:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772880.808712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772880.872724:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772880.936721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772881.000694:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772881.000720:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772881.000734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772881.064702:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772881.128715:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772881.128717:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772881.128730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772881.256715:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772881.256715:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772881.256733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772881.256738:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772881.320719:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772881.320720:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772881.320724:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772881.320731:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772881.320750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772881.320756:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772881.448705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772881.512715:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772881.576713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772881.640719:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772881.640719:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772881.640733:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772881.640753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772881.704714:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772881.704738:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772881.704814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772881.832705:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772881.832708:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772881.896716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772881.960705:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772882.024723:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772882.024725:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772882.024737:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772882.088706:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772882.152757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772882.152758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772882.152765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772882.280735:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772882.280743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772882.280743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772882.280746:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772882.344709:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772882.344728:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772882.344732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772882.344738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772882.344750:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772882.344754:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772882.472705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772882.536740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772882.600726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772882.664721:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772882.664722:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772882.664727:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772882.664733:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772882.728735:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772882.728746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772882.728778:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772882.856710:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772882.856713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772882.920719:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772882.984723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772883.048734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772883.048734:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772883.048734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772883.112729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772883.176725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772883.176728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772883.176738:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772883.304716:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772883.304721:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772883.304724:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772883.304734:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772883.368723:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772883.368723:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772883.368732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772883.368740:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772883.368757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772883.368767:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772883.496725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772883.560714:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772883.624724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772883.688729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772883.688739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772883.688740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772883.688740:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772883.752764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772883.752782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772883.752784:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772883.880711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772883.880712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772883.944715:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772884.008787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772884.072744:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772884.072744:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772884.072755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772884.136731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772884.200722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772884.200724:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772884.200727:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772884.328708:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772884.328720:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772884.328729:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772884.328781:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772884.392729:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772884.392741:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772884.392741:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772884.392769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772884.392773:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772884.392799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772884.520745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772884.584701:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772884.648724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772884.712743:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772884.712743:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772884.712747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772884.712784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772884.776743:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772884.776822:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772884.776875:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772884.904725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772884.904732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772884.968740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772885.032793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772885.096744:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772885.096749:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772885.096761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772885.160708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772885.224734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772885.224768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772885.224781:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772885.352723:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772885.352724:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772885.352727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772885.352775:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772885.416724:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772885.416767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772885.416769:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772885.416773:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772885.416783:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772885.416810:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772885.544710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772885.608708:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772885.672720:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772885.672759:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772885.672769:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772885.672774:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772885.672784:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1881 00000100:00080000:2.0:1790772885.672794:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772885.672806:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772885.672844:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534551808/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772885.672876:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772885.672881:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772885.672883:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772885.672889:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1881 00000100:00080000:2.0:1790772885.672894:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772885.672901:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772885.672912:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772885.672926:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534551808:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772885.672930:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534551936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772885.672949:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772885.672954:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772885.672958:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772885.672964:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1881 00000100:00080000:2.0:1790772885.672970:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772885.672977:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772885.672984:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772885.672998:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534552064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772885.673009:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790772885.673017:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772885.673024:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772885.673028:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772885.673036:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1881 00000100:00080000:2.0:1790772885.673044:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772885.673051:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772885.673058:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772885.673072:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772885/real 1790772885] req@ffff9c4efbcc0380 x1877759534551808/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772885.673078:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534552192/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772885.673106:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772885.673108:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534551808/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772885.673123:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772885.673130:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772885.673133:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772885.673141:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772885.673141:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1881 00000100:00100000:0.0:1790772885.673150:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534551808:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772885.673151:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772885.673165:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772885.673171:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772885.673180:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534551936:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772885.673190:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534552320/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790772885.673201:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772885.673208:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772885.673214:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1790772885.673215:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772885/real 1790772885] req@ffff9c4efbcc3100 x1877759534551936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772885.673218:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772885.673233:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1881 00000100:00080000:2.0:1790772885.673243:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772885.673253:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772885.673261:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772885.673268:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534551936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772885.673279:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534552448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772885.673305:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772885.673311:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772885.673312:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772885.673318:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534551936:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772885.673322:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772885.673327:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790772885.673328:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534552064:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772885.673343:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1881 00000100:00080000:2.0:1790772885.673350:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772885.673359:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772885.673365:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790772885.673367:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772885.673381:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772885/real 1790772885] req@ffff9c4efbcc0a80 x1877759534552064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772885.673390:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534552576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772885.673398:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534552064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772885.673440:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772885.673441:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772885.673449:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772885.673455:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1790772885.673456:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534552064:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772885.673465:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790772885.673467:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534552192:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772885.673482:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1881 00000100:00080000:2.0:1790772885.673494:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790772885.673500:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772885.673504:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772885.673513:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772885/real 1790772885] req@ffff9c4efbcc0700 x1877759534552192/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790772885.673513:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772885.673540:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534552704/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772885.673543:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534552192/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772885.673612:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772885.673616:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772885.673620:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790772885.673624:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772885.673626:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790772885.673629:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534552192:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772885.673639:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1881 00000100:00080000:2.0:1790772885.673646:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772885.673676:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534552320:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772885.673691:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790772885.673698:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790772885.673700:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772885.673713:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772885/real 1790772885] req@ffff9c4efbcc2680 x1877759534552320/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772885.673723:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534552832/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772885.673727:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534552320/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772885.673781:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772885.673790:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772885.673794:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534552320:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772885.673802:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534552448:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772885.673818:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772885.673827:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772885/real 1790772885] req@ffff9c4efbcc2300 x1877759534552448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772885.673835:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534552448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772885.673846:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772885.673848:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772885.673852:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534552448:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772885.673856:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534552576:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772885.673867:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772885.673874:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772885/real 1790772885] req@ffff9c4efbcc1880 x1877759534552576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772885.673886:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534552576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772885.673895:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772885.673899:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772885.673904:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534552576:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772885.673914:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534552704:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772885.673938:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772885.673948:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772885/real 1790772885] req@ffff9c4efbcc1500 x1877759534552704/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772885.673956:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534552704/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772885.673966:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772885.673971:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772885.673974:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534552704:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772885.673990:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534552832:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772885.674003:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772885.674011:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772885/real 1790772885] req@ffff9c4efbcc1c00 x1877759534552832/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772885.674019:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534552832/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772885.674026:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772885.674028:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772885.674030:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534552832:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772885.736730:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772885.736736:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772885.736776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772885.736778:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772885.800774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772885.800783:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772885.800817:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772885.928711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772885.928713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772885.992737:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772886.056798:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772886.120718:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772886.120750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772886.120751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772886.184706:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772886.248741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772886.248741:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772886.248745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772886.376721:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772886.376750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772886.376753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772886.376772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772886.440800:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772886.440800:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772886.440801:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772886.440832:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772886.440852:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772886.440865:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772886.568722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772886.632700:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772886.696724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772886.760707:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772886.760732:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772886.760741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772886.760777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772886.824738:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772886.824756:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772886.824851:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772886.952753:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772886.952753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772887.016746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772887.080710:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772887.144752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772887.144776:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772887.144784:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772887.208709:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772887.272730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772887.272730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772887.272735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772887.400743:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772887.400747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772887.400747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772887.400748:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772887.464749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772887.464753:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772887.464753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772887.464754:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772887.464820:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772887.464821:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772887.592714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772887.656752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772887.720723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772887.784745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772887.784745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772887.784762:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772887.784762:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772887.848752:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772887.848796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772887.848800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772887.976788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772887.976788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772888.040721:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772888.104741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772888.168717:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772888.168721:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772888.168766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772888.232738:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772888.296758:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772888.296758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772888.296758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772888.424722:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772888.424726:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772888.424727:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772888.424766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772888.488755:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772888.488755:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772888.488755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772888.488796:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772888.488797:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772888.488819:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772888.616808:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772888.680737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772888.744757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772888.808700:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772888.808728:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772888.808787:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772888.808815:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772888.872754:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772888.872794:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772888.872816:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772889.000731:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772889.000736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772889.064727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772889.128756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772889.192722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772889.192734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772889.192760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772889.256717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772889.320727:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772889.320733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772889.320735:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772889.448712:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772889.448727:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772889.448785:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772889.448800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772889.512719:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772889.512728:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772889.512762:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772889.512781:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772889.512788:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772889.512814:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772889.640726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772889.704751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772889.768722:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772889.832718:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772889.832726:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772889.832741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772889.832770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772889.896767:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772889.896767:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772889.896859:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772890.024723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772890.024724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772890.088715:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772890.152724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772890.216715:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772890.216760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772890.216779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772890.280760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772890.344718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772890.344739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772890.344740:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772890.472732:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772890.472747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772890.472784:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772890.472785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772890.536718:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772890.536723:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772890.536745:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772890.536783:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772890.536790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772890.536802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772890.664708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772890.728749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772890.792727:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772890.792732:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772890.792740:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772890.792745:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772890.792754:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1886 00000100:00080000:3.0:1790772890.792766:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772890.792787:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772890.792834:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534552960/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772890.792858:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772890.792861:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772890.792863:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772890.792866:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1886 00000100:00080000:3.0:1790772890.792869:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772890.792872:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772890.792882:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772890.792895:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534553088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772890.792905:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534552960:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772890.792906:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772890.792913:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772890.792916:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772890.792921:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1886 00000100:00080000:3.0:1790772890.792923:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772890.792926:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772890.792929:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772890.792938:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534553216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772890.792945:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772890.792948:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772890.792950:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772890.792953:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1886 00000100:00080000:3.0:1790772890.792955:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772890.792959:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772890.792962:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772890.792970:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534553344/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772890.792978:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772890.792981:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772890.792982:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772890.792986:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1886 00000100:00080000:3.0:1790772890.792989:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772890.792994:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790772890.792998:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790772890.793000:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772890.793009:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534553472/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772890.793023:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772890.793026:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772890.793028:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772890.793031:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1886 00000100:00080000:3.0:1790772890.793034:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772890.793037:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772890.793041:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772890.793050:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534553600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772890.793057:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772890/real 1790772890] req@ffff9c4ef73c2d80 x1877759534552960/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772890.793060:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772890.793067:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772890.793071:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772890.793079:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1886 00000100:00080000:3.0:1790772890.793082:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772890.793086:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772890.793089:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772890.793094:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534552960/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772890.793101:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534553728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772890.793120:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790772890.793121:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772890.793124:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772890.793127:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772890.793131:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772890.793134:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1886 00000100:00080000:3.0:1790772890.793138:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772890.793142:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534552960:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772890.793144:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772890.793153:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772890.793161:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534553856/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772890.793167:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772890.793170:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772890.793172:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772890.793174:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1886 00000100:00080000:3.0:1790772890.793177:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772890.793182:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772890.793183:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534553088:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772890.793186:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790772890.793214:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534553984/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790772890.793236:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772890.793254:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772890/real 1790772890] req@ffff9c4ef73c1500 x1877759534553088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772890.793268:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534553088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772890.793287:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772890.793292:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772890.793296:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534553088:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772890.793303:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534553216:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772890.793324:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772890.793336:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772890/real 1790772890] req@ffff9c4ef73c0a80 x1877759534553216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772890.793350:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534553216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772890.793360:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772890.793365:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772890.793369:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534553216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772890.793376:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534553344:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772890.793396:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772890.793407:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772890/real 1790772890] req@ffff9c4ef73c3800 x1877759534553344/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772890.793420:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534553344/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772890.793437:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772890.793441:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772890.793445:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534553344:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772890.793452:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534553472:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772890.793474:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772890.793486:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772890/real 1790772890] req@ffff9c4ef73c2a00 x1877759534553472/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772890.793499:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534553472/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772890.793509:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772890.793514:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772890.793518:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534553472:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772890.793524:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534553600:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772890.793547:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772890.793559:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772890/real 1790772890] req@ffff9c4ef73c0700 x1877759534553600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772890.793571:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534553600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772890.793589:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772890.793592:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772890.793597:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534553600:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772890.793604:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534553728:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772890.793624:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772890.793636:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772890/real 1790772890] req@ffff9c4ef73c2300 x1877759534553728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772890.793648:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534553728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772890.793680:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772890.793684:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772890.793688:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534553728:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772890.793695:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534553856:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772890.793716:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772890.793728:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772890/real 1790772890] req@ffff9c4ef73c0000 x1877759534553856/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772890.793742:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534553856/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772890.793753:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772890.793756:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772890.793760:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534553856:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772890.793783:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534553984:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772890.793804:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772890.793816:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772890/real 1790772890] req@ffff9c4ef73c1c00 x1877759534553984/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772890.793828:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534553984/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772890.793838:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772890.793842:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772890.793845:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534553984:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772890.856739:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772890.856739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772890.856739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772890.856740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772890.920757:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772890.920772:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790772890.920778:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772891.048718:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772891.048718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772891.112717:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772891.176803:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772891.240731:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772891.240731:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772891.240731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772891.304721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772891.368724:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772891.368726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772891.368742:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772891.496751:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772891.496751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772891.496752:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772891.496752:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772891.560712:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772891.560716:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772891.560725:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772891.560729:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772891.560750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772891.560770:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772891.688721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772891.752777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772891.816713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772891.880720:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772891.880722:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772891.880724:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772891.880725:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772891.944718:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772891.944732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772891.944794:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772892.072728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772892.072749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772892.136715:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772892.200714:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772892.264737:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772892.264760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772892.265691:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772892.328714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772892.392773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772892.392803:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772892.392869:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772892.520712:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772892.520730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772892.520735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772892.520771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772892.584708:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772892.584729:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772892.584732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772892.584749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772892.584753:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772892.584766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772892.712749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772892.776704:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772892.840716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772892.904734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772892.904734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772892.904739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772892.904740:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772892.968705:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772892.968748:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772892.968770:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772893.096713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772893.096722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772893.160758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772893.224725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772893.288706:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772893.288706:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772893.288714:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772893.352724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772893.416721:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772893.416751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772893.416763:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772893.544743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772893.544743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772893.544744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772893.544778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772893.608761:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772893.608771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772893.608771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772893.608771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772893.608812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772893.608826:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772893.736740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772893.800746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772893.864716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772893.928730:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772893.928742:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772893.928746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772893.928748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772893.992727:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772893.992743:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772893.992860:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772894.120723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772894.120723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772894.184743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772894.248722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772894.312732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772894.312733:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772894.312738:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772894.376704:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772894.440738:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772894.440738:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772894.440776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772894.568741:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772894.568743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772894.568752:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772894.568785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772894.632733:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772894.632735:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772894.632742:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772894.632779:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772894.632801:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772894.632820:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772894.760725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772894.824727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772894.888718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772894.952743:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772894.952747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772894.952748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772894.952751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772895.016756:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772895.016764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772895.016819:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772895.144723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772895.144729:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772895.208719:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772895.272752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772895.336708:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772895.336739:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772895.336819:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772895.400713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772895.464779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772895.464783:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772895.464784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772895.592751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772895.592783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772895.592802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772895.592808:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772895.656711:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772895.656751:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772895.656771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772895.656795:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772895.656796:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772895.656803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772895.784702:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772895.848747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772895.912715:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772895.912780:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772895.912788:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772895.912792:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772895.912803:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1891 00000100:00080000:1.0:1790772895.912815:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772895.912829:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772895.912874:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534554112/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772895.912902:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772895.912906:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772895.912907:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772895.912912:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1891 00000100:00080000:1.0:1790772895.912915:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772895.912920:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772895.912932:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772895.912940:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534554112:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772895.912947:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534554240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772895.912958:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772895.912962:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772895.912964:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772895.912971:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1891 00000100:00080000:1.0:1790772895.912974:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772895.912977:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772895.912981:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772895.912989:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534554368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772895.912996:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772895.913000:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772895.913001:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772895.913005:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1891 00000100:00080000:1.0:1790772895.913008:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772895.913011:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772895.913016:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772895.913025:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534554496/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772895.913031:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772895.913033:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772895.913035:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772895.913038:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1891 00000100:00080000:1.0:1790772895.913049:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772895.913053:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772895.913058:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:0.0:1790772895.913062:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772895.913076:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534554624/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772895.913101:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772895.913107:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772895.913110:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772895.913118:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1891 00000100:00080000:1.0:1790772895.913123:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772895.913132:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772895.913139:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772895.913146:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772895/real 1790772895] req@ffff9c4efac4fb80 x1877759534554112/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772895.913158:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534554752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772895.913168:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534554112/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772895.913169:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772895.913189:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772895.913193:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772895.913214:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1891 00000100:00080000:0.0:1790772895.913221:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772895.913221:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772895.913237:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772895.913238:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772895.913250:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534554112:192.168.204.150@tcp:250:kworker.0 00000008:00100000:1.0:1790772895.913252:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772895.913272:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534554880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772895.913282:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534554240:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772895.913307:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772895.913307:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772895.913315:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772895.913319:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772895.913327:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1891 00000100:00080000:1.0:1790772895.913332:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772895.913336:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772895/real 1790772895] req@ffff9c4efac4e300 x1877759534554240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772895.913346:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772895.913355:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534554240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772895.913360:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790772895.913371:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772895.913376:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790772895.913381:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534555008/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772895.913382:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534554240:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772895.913404:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534554368:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772895.913409:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772895.913413:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772895.913416:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772895.913422:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1891 00000100:00080000:1.0:1790772895.913427:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790772895.913432:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772895.913438:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772895.913445:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772895/real 1790772895] req@ffff9c4efac4ea00 x1877759534554368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772895.913446:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772895.913469:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534555136/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772895.913471:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534554368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772895.913521:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772895.913525:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772895.913529:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534554368:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772895.913536:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534554496:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772895.913552:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772895.913558:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772895/real 1790772895] req@ffff9c4efac4e680 x1877759534554496/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772895.913567:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534554496/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772895.913574:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772895.913576:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772895.913578:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534554496:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772895.913582:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534554624:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772895.913592:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772895.913598:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772895/real 1790772895] req@ffff9c4efac4f480 x1877759534554624/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772895.913606:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534554624/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772895.913612:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772895.913614:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772895.913616:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534554624:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772895.913620:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534554752:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772895.913631:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772895.913637:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772895/real 1790772895] req@ffff9c4efac4f800 x1877759534554752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772895.913644:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534554752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772895.913679:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772895.913682:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772895.913685:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534554752:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772895.913699:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534554880:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772895.913713:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772895.913721:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772895/real 1790772895] req@ffff9c4efac4dc00 x1877759534554880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772895.913728:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534554880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772895.913734:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772895.913737:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772895.913739:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534554880:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772895.913743:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534555008:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772895.913753:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772895.913759:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772895/real 1790772895] req@ffff9c4efac4ed80 x1877759534555008/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772895.913766:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534555008/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772895.913773:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772895.913775:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772895.913778:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534555008:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772895.913782:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534555136:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772895.913792:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772895.913798:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772895/real 1790772895] req@ffff9c4efac4c700 x1877759534555136/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772895.913805:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534555136/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772895.913812:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772895.913814:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772895.913816:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534555136:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772895.976762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772895.976762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772895.976764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772895.976772:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772896.040726:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772896.040790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772896.040839:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772896.168716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772896.168716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772896.232743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772896.296750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772896.360701:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772896.360710:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772896.360715:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772896.424703:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772896.488732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772896.488762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772896.488762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772896.616740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772896.616740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772896.616769:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772896.616775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772896.680729:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772896.680740:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772896.680751:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772896.680766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772896.680788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772896.680811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772896.808710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772896.872717:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772896.936714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772897.000711:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772897.000722:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772897.000748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772897.000749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772897.064760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772897.064760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772897.064789:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772897.192711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772897.192735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772897.256782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772897.320726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772897.384789:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772897.384792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772897.384795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772897.448714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772897.512710:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772897.512717:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772897.512739:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772897.640745:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772897.640805:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772897.640805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772897.640806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772897.704766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772897.704767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772897.704798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772897.704803:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772897.704832:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772897.704849:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772897.832702:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772897.896733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772897.960719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772898.024721:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772898.024731:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772898.024762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772898.024769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772898.088740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772898.088774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772898.088879:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772898.216705:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772898.216721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772898.280759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772898.344804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772898.408771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772898.408780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772898.408786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772898.472716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772898.536769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772898.536775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772898.536776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772898.664725:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772898.664728:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772898.664729:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772898.664750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772898.728728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772898.728736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772898.728743:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772898.728777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772898.728783:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772898.728790:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772898.856750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772898.920716:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772898.984726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772899.048741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772899.048777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772899.048778:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772899.048779:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772899.112724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772899.112727:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772899.112875:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772899.240720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772899.240721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772899.304773:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772899.368781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772899.432753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772899.432753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772899.432753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772899.496750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772899.560714:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772899.560748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772899.560750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772899.688755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772899.688755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772899.688764:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772899.688764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772899.752738:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772899.752779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772899.752781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772899.752786:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772899.752802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772899.752842:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772899.880721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772899.944732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772900.008730:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772900.072775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772900.072775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772900.072784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772900.072785:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772900.136748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772900.136753:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772900.136852:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772900.264712:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772900.264717:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772900.328711:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772900.392744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772900.456714:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772900.456723:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772900.456725:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772900.520713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772900.584727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772900.584728:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772900.584730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772900.712729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772900.712745:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772900.712753:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772900.712771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772900.776728:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772900.776729:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772900.776731:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772900.776746:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772900.776767:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772900.776786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772900.904705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772900.968697:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772901.032725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772901.032727:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772901.032738:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772901.032743:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772901.032754:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1896 00000100:00080000:1.0:1790772901.032766:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772901.032778:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772901.032823:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534555264/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772901.032851:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772901.032854:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772901.032856:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772901.032859:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1896 00000100:00080000:1.0:1790772901.032862:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772901.032865:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772901.032874:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772901.032885:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534555392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772901.032893:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772901.032896:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772901.032897:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772901.032900:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1896 00000100:00080000:1.0:1790772901.032902:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772901.032905:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772901.032909:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772901.032916:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534555520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772901.032923:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772901.032925:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772901.032926:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772901.032930:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1896 00000100:00080000:1.0:1790772901.032932:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772901.032935:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772901.032938:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772901.032945:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534555648/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772901.032952:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772901.032956:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772901.032959:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772901.032962:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1896 00000100:00080000:1.0:1790772901.032965:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772901.032968:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772901.032971:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772901.032978:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534555776/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772901.032986:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534555264:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772901.033004:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772901.033008:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772901.033010:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772901.033015:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1896 00000100:00080000:1.0:1790772901.033018:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772901.033026:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772901.033031:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772901.033042:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534555904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772901.033049:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772901.033052:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772901.033053:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772901.033058:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1896 00000100:00080000:1.0:1790772901.033061:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772901.033064:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772901.033068:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790772901.033073:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772901.033082:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534556032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772901.033094:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772901.033097:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772901.033099:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772901.033102:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1896 00000100:00080000:1.0:1790772901.033106:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772901.033111:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772901.033116:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772901.033120:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772901/real 1790772901] req@ffff9c4efac4df80 x1877759534555264/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772901.033130:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534556160/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772901.033140:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772901.033142:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534555264/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772901.033144:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772901.033149:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772901.033160:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1896 00000100:00080000:1.0:1790772901.033177:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772901.033181:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772901.033185:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790772901.033194:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790772901.033195:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772901.033205:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534555264:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772901.033212:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534556288/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772901.033221:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534555392:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772901.033256:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772901.033265:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772901/real 1790772901] req@ffff9c4efac4c700 x1877759534555392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772901.033274:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534555392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772901.033288:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772901.033291:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772901.033294:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534555392:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772901.033299:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534555520:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772901.033310:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772901.033317:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772901/real 1790772901] req@ffff9c4efac4ed80 x1877759534555520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772901.033325:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534555520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772901.033330:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772901.033333:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772901.033335:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534555520:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772901.033339:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534555648:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772901.033351:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772901.033366:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772901/real 1790772901] req@ffff9c4efac4dc00 x1877759534555648/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772901.033374:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534555648/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772901.033381:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772901.033384:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772901.033386:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534555648:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772901.033409:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534555776:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772901.033422:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772901.033428:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772901/real 1790772901] req@ffff9c4efac4f800 x1877759534555776/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772901.033436:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534555776/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772901.033442:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772901.033444:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772901.033446:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534555776:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772901.033451:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534555904:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772901.033461:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772901.033468:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772901/real 1790772901] req@ffff9c4efac4f480 x1877759534555904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772901.033480:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534555904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772901.033491:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772901.033493:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772901.033495:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534555904:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772901.033499:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534556032:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772901.033509:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772901.033516:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772901/real 1790772901] req@ffff9c4efac4e680 x1877759534556032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772901.033524:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534556032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772901.033529:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772901.033531:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772901.033533:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534556032:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772901.033538:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534556160:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772901.033548:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772901.033554:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772901/real 1790772901] req@ffff9c4efac4ea00 x1877759534556160/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772901.033561:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534556160/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772901.033573:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772901.033576:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772901.033578:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534556160:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772901.033582:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534556288:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772901.033592:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772901.033598:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772901/real 1790772901] req@ffff9c4efac4e300 x1877759534556288/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772901.033605:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534556288/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772901.033612:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772901.033614:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772901.033616:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534556288:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772901.096755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772901.096766:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772901.096769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772901.096770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772901.160785:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772901.160794:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772901.160836:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772901.288731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772901.288744:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772901.352732:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772901.416741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772901.480703:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772901.480726:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772901.480730:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772901.544730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772901.608742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772901.608742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772901.608749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772901.736734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772901.736737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772901.736769:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772901.736781:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772901.800738:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772901.800746:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772901.800757:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772901.800759:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772901.800796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772901.800801:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772901.928710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772901.992739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772902.056758:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772902.120739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772902.120751:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772902.120752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772902.120757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772902.184800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772902.184837:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772902.184860:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772902.312770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772902.312789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772902.376749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772902.440736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772902.504728:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772902.504728:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772902.504750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772902.568708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772902.632726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772902.632730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772902.632734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772902.760747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772902.760747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772902.760748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772902.760810:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772902.824729:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772902.824739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772902.824744:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772902.824789:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772902.824791:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772902.824793:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772902.952752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772903.016717:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772903.080708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772903.144749:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772903.144750:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772903.144750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772903.144750:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772903.208718:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772903.208742:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772903.208801:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772903.336716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772903.336724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772903.400770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772903.464713:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772903.528723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772903.528723:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772903.528730:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772903.592750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772903.656753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772903.656757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772903.656768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772903.784723:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772903.784723:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772903.784734:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772903.784759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772903.848714:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772903.848717:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772903.848731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772903.848773:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772903.848793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772903.848796:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772903.976701:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772904.040718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772904.104719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772904.168723:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772904.168781:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772904.168786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772904.168786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772904.232739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772904.232746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772904.232923:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772904.360712:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772904.360735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772904.424749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772904.488717:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772904.552732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772904.552735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772904.552749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772904.616716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772904.680725:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772904.680729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772904.680773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772904.808757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772904.808762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772904.808806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772904.808806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772904.872736:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772904.872759:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772904.872783:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772904.872787:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772904.872811:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772904.872817:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772905.000703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772905.064734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772905.128741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772905.192812:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772905.192812:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772905.192817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772905.192819:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772905.256760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772905.256767:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772905.256862:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772905.384763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772905.384771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772905.448743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772905.512789:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772905.576761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772905.576774:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772905.576777:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772905.640742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772905.704830:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772905.704830:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772905.704832:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772905.832734:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772905.832734:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772905.832771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772905.832771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772905.896744:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772905.896746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772905.896757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772905.896757:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772905.896792:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772905.896792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772906.024728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772906.088726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772906.152757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772906.152762:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772906.152768:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772906.152772:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772906.152782:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1902 00000100:00080000:3.0:1790772906.152790:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772906.152800:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772906.152834:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534556416/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772906.152857:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772906.152861:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772906.152862:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772906.152866:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1902 00000100:00080000:3.0:1790772906.152869:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772906.152873:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772906.152881:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772906.152905:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534556416:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772906.152906:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534556544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772906.152921:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772906.152924:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772906.152926:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772906.152929:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1902 00000100:00080000:3.0:1790772906.152933:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772906.152938:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772906.152942:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772906.152951:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534556672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772906.152964:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772906.152967:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772906.152968:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790772906.152974:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772906.152975:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1902 00000100:00080000:3.0:1790772906.152981:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772906.152984:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772906.152988:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772906.152997:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534556800/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772906.153004:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772906.153006:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772906.153007:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772906.153010:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1902 00000100:00080000:3.0:1790772906.153013:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772906.153016:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772906.153022:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772906.153040:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772906/real 1790772906] req@ffff9c4ef73c0e00 x1877759534556416/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772906.153041:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534556928/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772906.153072:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772906.153075:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772906.153077:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772906.153080:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1902 00000100:00100000:0.0:1790772906.153084:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534556416/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772906.153085:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772906.153098:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772906.153100:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790772906.153104:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772906.153110:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534556416:192.168.204.150@tcp:250:kworker.0 00000008:00100000:3.0:1790772906.153111:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772906.153129:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534557056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772906.153139:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534556544:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772906.153143:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772906.153149:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772906.153152:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772906.153159:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1902 00000100:00080000:3.0:1790772906.153163:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772906.153168:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790772906.153175:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790772906.153176:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772906.153189:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534557184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772906.153195:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772906/real 1790772906] req@ffff9c4ef73c1c00 x1877759534556544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772906.153201:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772906.153208:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772906.153211:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772906.153217:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1902 00000100:00100000:0.0:1790772906.153222:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534556544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772906.153222:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772906.153231:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772906.153238:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772906.153248:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534557312/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772906.153249:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772906.153258:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772906.153265:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534556544:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772906.153274:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772906.153278:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534556672:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772906.153279:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772906.153283:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772906.153287:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1902 00000100:00080000:3.0:1790772906.153290:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772906.153293:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772906.153297:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:0.0:1790772906.153303:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772906.153308:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534557440/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772906.153316:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772906/real 1790772906] req@ffff9c4ef73c0000 x1877759534556672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772906.153326:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534556672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772906.153333:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772906.153336:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772906.153339:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534556672:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772906.153344:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534556800:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772906.153357:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772906.153364:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772906/real 1790772906] req@ffff9c4ef73c2300 x1877759534556800/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772906.153372:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534556800/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772906.153379:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772906.153381:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772906.153383:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534556800:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772906.153386:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534556928:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772906.153397:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772906.153403:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772906/real 1790772906] req@ffff9c4ef73c0700 x1877759534556928/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772906.153410:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534556928/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772906.153416:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772906.153418:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772906.153420:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534556928:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772906.153438:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534557056:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772906.153459:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772906.153472:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772906/real 1790772906] req@ffff9c4ef73c2a00 x1877759534557056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772906.153485:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534557056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772906.153498:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772906.153500:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772906.153502:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534557056:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772906.153507:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534557184:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772906.153531:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772906.153540:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772906/real 1790772906] req@ffff9c4ef73c3800 x1877759534557184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772906.153548:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534557184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772906.153554:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772906.153556:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772906.153558:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534557184:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772906.153562:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534557312:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772906.153573:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772906.153579:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772906/real 1790772906] req@ffff9c4ef73c0a80 x1877759534557312/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772906.153587:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534557312/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772906.153600:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772906.153602:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772906.153604:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534557312:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772906.153608:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534557440:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772906.153618:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772906.153624:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772906/real 1790772906] req@ffff9c4ef73c1500 x1877759534557440/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772906.153631:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534557440/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772906.153637:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772906.153639:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772906.153641:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534557440:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772906.216719:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772906.216729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772906.216749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772906.216757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772906.280791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772906.280861:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772906.280903:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772906.408719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772906.408773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772906.472790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772906.536769:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772906.600800:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772906.600801:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772906.600801:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772906.664743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772906.728746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772906.728775:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772906.728786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772906.856765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772906.856775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772906.856804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772906.856809:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772906.920742:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772906.920743:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772906.920826:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772906.920829:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772906.920853:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772906.920854:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772907.048751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772907.112757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772907.176721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772907.240752:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772907.240757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772907.240778:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772907.240802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772907.304779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772907.304783:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772907.304820:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772907.432731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772907.432736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772907.496724:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772907.560753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772907.624719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772907.624740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772907.624779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772907.688736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772907.752746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772907.752747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772907.752747:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772907.880728:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772907.880732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772907.880735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772907.880781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772907.944765:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772907.944769:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772907.944781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772907.944782:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772907.944814:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772907.944820:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772908.072748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772908.136743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772908.200772:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772908.264698:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772908.264750:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772908.264755:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772908.264755:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772908.328737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772908.328827:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772908.328863:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772908.456707:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772908.456715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772908.520708:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772908.584813:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772908.648776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772908.648833:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772908.648883:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772908.712786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772908.776785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772908.776801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772908.776816:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772908.904763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772908.904829:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772908.904857:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772908.904880:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772908.968799:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772908.968804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772908.968896:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772908.968911:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772908.968927:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772908.968930:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772909.096758:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772909.160787:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772909.224724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772909.288777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772909.288778:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772909.288786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772909.288787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772909.352723:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772909.352730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772909.352891:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772909.480733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772909.480753:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772909.544749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772909.608756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772909.672719:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772909.672753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772909.672753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772909.736728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772909.800801:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772909.800810:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772909.800812:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772909.928769:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772909.928770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772909.928779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772909.928809:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772909.992732:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772909.992750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772909.992773:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772909.992773:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772909.992805:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772909.992808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772910.120714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772910.184807:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772910.248713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772910.312798:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772910.312824:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772910.312824:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772910.312827:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772910.376777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772910.376824:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772910.376942:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772910.504746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772910.504756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772910.568733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772910.632773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772910.696757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772910.696790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772910.696792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772910.760726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772910.824722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772910.824725:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772910.824734:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772910.952765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772910.952765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772910.952771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772910.952771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772911.016723:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772911.016744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772911.016751:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772911.016758:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772911.016782:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772911.016809:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772911.144703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772911.208732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772911.272723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772911.272730:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772911.272741:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772911.272747:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772911.272758:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1907 00000100:00080000:3.0:1790772911.272770:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772911.272783:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772911.272838:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534557568/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772911.272864:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772911.272867:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772911.272868:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772911.272872:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1907 00000100:00080000:3.0:1790772911.272877:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772911.272884:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772911.272893:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772911.272906:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534557568:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772911.272908:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534557696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772911.272926:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772911.272928:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772911.272929:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772911.272937:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1907 00000100:00080000:3.0:1790772911.272941:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772911.272946:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772911.272952:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772911.272966:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534557824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772911.272978:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772911.272983:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772911.272985:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772911.272991:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1907 00000100:00080000:3.0:1790772911.272997:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790772911.273006:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772911.273007:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772911.273018:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772911.273033:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534557952/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772911.273042:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772911.273045:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772911.273047:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772911.273051:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1907 00000100:00080000:3.0:1790772911.273058:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772911.273066:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772911.273069:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772911.273071:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772911/real 1790772911] req@ffff9c4ef73c2d80 x1877759534557568/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772911.273082:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534558080/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772911.273099:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772911.273102:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1790772911.273109:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534557568/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772911.273111:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772911.273129:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1907 00000100:00080000:3.0:1790772911.273141:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772911.273145:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772911.273149:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790772911.273156:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:3.0:1790772911.273162:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772911.273167:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534557568:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772911.273179:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534558208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772911.273189:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772911.273192:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772911.273193:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772911.273199:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1907 00000100:00100000:0.0:1790772911.273203:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534557696:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772911.273205:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772911.273217:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772911.273223:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790772911.273245:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772911.273247:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534558336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772911.273258:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772911/real 1790772911] req@ffff9c4ef73c1500 x1877759534557696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772911.273268:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772911.273274:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1790772911.273277:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534557696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772911.273278:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772911.273296:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1907 00000100:00080000:3.0:1790772911.273311:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772911.273319:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790772911.273326:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790772911.273326:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790772911.273339:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772911.273344:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534557696:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772911.273348:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534558464/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772911.273359:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534557824:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772911.273369:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772911.273374:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772911.273378:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790772911.273381:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772911.273394:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1907 00000100:00100000:0.0:1790772911.273396:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772911/real 1790772911] req@ffff9c4ef73c0a80 x1877759534557824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772911.273404:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772911.273410:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534557824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772911.273423:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790772911.273428:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790772911.273430:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790772911.273435:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772911.273440:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534557824:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772911.273447:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534558592/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772911.273454:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534557952:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772911.273490:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772911.273506:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772911/real 1790772911] req@ffff9c4ef73c3800 x1877759534557952/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772911.273515:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534557952/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772911.273522:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772911.273525:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772911.273527:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534557952:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772911.273532:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534558080:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772911.273545:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772911.273552:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772911/real 1790772911] req@ffff9c4ef73c2a00 x1877759534558080/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772911.273559:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534558080/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772911.273566:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772911.273568:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772911.273570:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534558080:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772911.273575:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534558208:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772911.273586:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772911.273593:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772911/real 1790772911] req@ffff9c4ef73c0700 x1877759534558208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772911.273601:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534558208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772911.273613:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772911.273615:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772911.273617:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534558208:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772911.273633:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534558336:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772911.273645:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772911.273671:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772911/real 1790772911] req@ffff9c4ef73c2300 x1877759534558336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772911.273681:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534558336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772911.273687:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772911.273690:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772911.273692:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534558336:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772911.273697:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534558464:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772911.273709:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772911.273716:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772911/real 1790772911] req@ffff9c4ef73c0000 x1877759534558464/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772911.273724:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534558464/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772911.273736:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772911.273739:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772911.273741:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534558464:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772911.273745:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534558592:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772911.273755:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772911.273761:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772911/real 1790772911] req@ffff9c4ef73c1c00 x1877759534558592/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772911.273769:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534558592/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772911.273776:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772911.273778:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772911.273780:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534558592:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772911.336756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772911.336757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772911.336767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772911.336769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772911.400719:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772911.400751:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772911.400899:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772911.528717:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772911.528721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772911.592730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772911.656728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772911.720734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772911.720740:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772911.720815:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772911.784731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772911.848724:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772911.848774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772911.848794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772911.976768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772911.976821:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772911.976843:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772911.976843:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772912.040736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772912.040736:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772912.040748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772912.040786:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772912.040799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772912.040820:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772912.168699:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772912.232757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772912.296707:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772912.360754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772912.360754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772912.360754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772912.360782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772912.424714:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772912.424755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772912.424874:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772912.552748:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772912.552757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772912.616725:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772912.680833:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772912.744774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772912.744790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772912.744793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772912.808725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772912.872730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772912.872734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772912.872776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772913.000747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772913.000747:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772913.000806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772913.000835:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772913.064742:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772913.064804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772913.064807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772913.064826:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772913.064830:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772913.064874:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772913.192731:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772913.256719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772913.320727:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772913.384731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772913.384738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772913.384760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772913.384781:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772913.448715:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772913.448783:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772913.448901:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772913.576722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772913.576722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772913.640732:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772913.704729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772913.768761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772913.768771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772913.768774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772913.832754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772913.896720:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772913.896755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772913.896760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772914.024749:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772914.024756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772914.024765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772914.024797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772914.088751:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772914.088756:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772914.088783:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772914.088798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772914.088815:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772914.088820:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772914.216715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772914.280725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772914.344704:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772914.408762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772914.408765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772914.408780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772914.408786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772914.472755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772914.472765:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772914.472887:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772914.600744:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772914.600749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772914.664716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772914.728757:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772914.792757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772914.792757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772914.792783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772914.856742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772914.920747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772914.920750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772914.920754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772915.048761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772915.048770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772915.048770:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772915.048771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772915.112737:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772915.112742:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772915.112750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772915.112767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772915.112797:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772915.112797:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772915.240724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772915.304750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772915.368702:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772915.432731:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772915.432736:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772915.432747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772915.432747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772915.496742:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772915.496762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772915.496812:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772915.624735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772915.624735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772915.688767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772915.752727:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772915.816734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772915.816737:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772915.816755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772915.880718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772915.944749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772915.944750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772915.944754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772916.072755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772916.072759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772916.072759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772916.072759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772916.136735:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772916.136735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772916.136746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772916.136805:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772916.136811:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772916.136816:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772916.264725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772916.328738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772916.392758:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772916.392765:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772916.392774:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772916.392777:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772916.392786:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1912 00000100:00080000:3.0:1790772916.392795:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772916.392814:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772916.392851:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534558720/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772916.392875:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772916.392878:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772916.392880:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772916.392883:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1912 00000100:00080000:3.0:1790772916.392887:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772916.392892:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772916.392901:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772916.392919:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534558720:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772916.392920:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534558848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772916.392948:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772916.392952:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772916.392955:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772916.392962:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1912 00000100:00080000:3.0:1790772916.392968:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772916.392974:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772916.392982:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772916.392998:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534558976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790772916.393006:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772916.393017:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772916.393021:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772916.393024:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772916.393028:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1912 00000100:00080000:3.0:1790772916.393032:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772916.393038:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772916.393042:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772916.393053:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534559104/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772916.393061:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772916.393062:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772916/real 1790772916] req@ffff9c4ef73c0e00 x1877759534558720/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772916.393071:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772916.393079:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772916.393097:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1912 00000100:00080000:3.0:1790772916.393102:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772916.393109:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772916.393114:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534558720/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772916.393115:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772916.393136:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534559232/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772916.393169:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772916.393170:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772916.393186:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790772916.393188:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772916.393191:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790772916.393198:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534558720:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772916.393204:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1912 00000100:00080000:3.0:1790772916.393208:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772916.393212:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772916.393217:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772916.393226:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534559360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772916.393232:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534558848:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772916.393255:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772916.393258:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772916.393260:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772916.393265:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1912 00000100:00080000:3.0:1790772916.393269:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790772916.393276:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772916.393280:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772916.393285:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772916.393297:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772916/real 1790772916] req@ffff9c4ef73c1c00 x1877759534558848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772916.393301:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534559488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772916.393336:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534558848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772916.393352:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772916.393372:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772916.393375:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772916.393380:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1912 00000100:00080000:3.0:1790772916.393385:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772916.393386:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772916.393395:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772916.393395:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772916.393410:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534558848:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772916.393411:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772916.393424:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534558976:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772916.393433:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534559616/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772916.393442:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790772916.393444:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772916.393452:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772916.393456:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790772916.393467:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772916/real 1790772916] req@ffff9c4ef73c0000 x1877759534558976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772916.393472:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1912 00000100:00080000:3.0:1790772916.393488:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772916.393493:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534558976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772916.393500:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772916.393516:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790772916.393518:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772916.393525:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790772916.393529:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534559744/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772916.393530:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534558976:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772916.393552:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534559104:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772916.393577:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772916.393586:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772916/real 1790772916] req@ffff9c4ef73c2300 x1877759534559104/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772916.393594:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534559104/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772916.393602:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772916.393604:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772916.393607:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534559104:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772916.393611:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534559232:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772916.393622:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772916.393628:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772916/real 1790772916] req@ffff9c4ef73c0700 x1877759534559232/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772916.393635:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534559232/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772916.393642:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772916.393644:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772916.393646:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534559232:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772916.393681:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534559360:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772916.393718:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772916.393731:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772916/real 1790772916] req@ffff9c4ef73c2a00 x1877759534559360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772916.393746:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534559360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772916.393761:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772916.393764:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772916.393767:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534559360:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772916.393772:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534559488:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772916.393788:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772916.393796:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772916/real 1790772916] req@ffff9c4ef73c3800 x1877759534559488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772916.393805:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534559488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772916.393811:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772916.393814:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772916.393816:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534559488:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772916.393820:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534559616:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772916.393830:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772916.393836:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772916/real 1790772916] req@ffff9c4ef73c0a80 x1877759534559616/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772916.393844:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534559616/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772916.393857:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772916.393860:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772916.393862:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534559616:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772916.393866:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534559744:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772916.393876:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772916.393882:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772916/real 1790772916] req@ffff9c4ef73c1500 x1877759534559744/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772916.393889:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534559744/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772916.393895:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772916.393897:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772916.393899:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534559744:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772916.456735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772916.456741:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772916.456753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772916.456767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772916.520749:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772916.520760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772916.520817:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772916.648745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772916.648758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772916.712751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772916.776751:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772916.840725:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772916.840758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772916.840758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772916.904744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772916.968734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772916.968744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772916.968767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772917.096746:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772917.096754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772917.096778:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772917.096778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772917.160734:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772917.160755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772917.160798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772917.160814:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772917.160816:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772917.160818:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772917.288733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772917.352736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772917.416719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772917.480751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772917.480783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772917.480786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772917.480794:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772917.544758:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772917.544802:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772917.544858:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772917.672733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772917.672772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772917.736759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772917.800745:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772917.864778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772917.864778:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772917.864782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772917.928729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772917.992747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772917.992760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772917.992785:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772918.120731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772918.120736:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772918.120747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772918.120771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772918.184729:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772918.184736:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772918.184770:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772918.184772:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772918.184782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772918.184808:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772918.312725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772918.376728:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772918.440728:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772918.504747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772918.504750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772918.504753:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772918.504775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772918.568749:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772918.568791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772918.568911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772918.696713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772918.696728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772918.760792:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772918.824728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772918.888740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772918.888740:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772918.888751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772918.952723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772919.016722:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772919.016722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772919.016758:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772919.144728:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772919.144728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772919.144739:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772919.144782:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772919.208777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772919.208778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772919.208790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772919.208790:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772919.208826:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772919.208845:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772919.336718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772919.400726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772919.464718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772919.528741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772919.528757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772919.528774:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772919.528776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772919.592713:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772919.592729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772919.592871:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772919.720728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772919.720750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772919.784739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772919.848803:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772919.912779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772919.912779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772919.912783:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772919.976733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772920.040719:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772920.040732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772920.040737:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772920.168743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772920.168744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772920.168752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772920.168790:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772920.232719:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772920.232737:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772920.232756:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772920.232775:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772920.232776:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772920.232784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772920.360696:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772920.424723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772920.488702:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772920.552771:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772920.552771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772920.552772:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772920.552805:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772920.616716:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772920.616733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772920.616843:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772920.744708:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772920.744748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772920.808727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772920.872721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772920.936713:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772920.936717:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772920.936772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772921.000706:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772921.064741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772921.064766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772921.064767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772921.192726:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772921.192728:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772921.192751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772921.192755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772921.256731:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772921.256760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772921.256760:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772921.256761:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772921.256790:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772921.256811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772921.384700:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772921.448729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772921.512734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772921.512740:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772921.512749:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772921.512755:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772921.512765:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1917 00000100:00080000:2.0:1790772921.512776:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772921.512789:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772921.512826:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534559872/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772921.512859:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772921.512864:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772921.512867:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772921.512872:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1917 00000100:00080000:2.0:1790772921.512876:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772921.512884:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772921.512894:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772921.512904:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534559872:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772921.512912:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534560000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772921.512925:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772921.512929:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772921.512932:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772921.512936:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1917 00000100:00080000:2.0:1790772921.512941:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772921.512947:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772921.512953:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772921.512968:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534560128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772921.512979:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772921.512984:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772921.512986:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772921.512991:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1917 00000100:00080000:2.0:1790772921.512997:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772921.513003:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772921.513010:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772921.513016:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534560000:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772921.513027:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534560256/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772921.513039:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772921.513044:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772921.513046:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790772921.513054:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534560128:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772921.513054:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1917 00000100:00080000:2.0:1790772921.513061:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772921.513071:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772921.513077:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772921.513092:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534560384/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772921.513094:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534560256:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772921.513114:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772921.513118:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772921.513121:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772921.513127:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1917 00000100:00080000:2.0:1790772921.513131:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772921.513135:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534560384:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772921.513141:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772921.513148:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772921.513163:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534560512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772921.513184:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772921.513188:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772921.513190:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772921.513195:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1917 00000100:00080000:2.0:1790772921.513199:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772921.513205:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772921.513210:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534560512:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790772921.513212:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772921.513230:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534560640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772921.513242:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772921.513246:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772921.513248:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772921.513252:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1917 00000100:00100000:0.0:1790772921.513256:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534560640:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772921.513258:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772921.513268:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772921.513274:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772921.513287:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534560768/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772921.513304:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772921.513308:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772921.513311:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772921.513316:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1917 00000100:00080000:2.0:1790772921.513321:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772921.513326:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772921.513332:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772921.513337:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534560768:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772921.513348:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534560896/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772921.513376:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534560896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790772921.576730:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772921.576742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772921.576769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772921.576770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772921.640762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772921.640789:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772921.640838:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772921.768706:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772921.768712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772921.832753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772921.896733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772921.960760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772921.960791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772921.960797:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772922.024727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772922.088723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772922.088745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772922.088772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772922.216769:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772922.216769:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772922.216783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772922.216787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772922.280724:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772922.280730:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772922.280771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772922.280773:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772922.280781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772922.280821:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772922.408732:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772922.472719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772922.536731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772922.600771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772922.600777:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772922.600796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772922.600807:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772922.664740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772922.664762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772922.664922:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772922.792737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772922.792746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772922.856775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772922.920713:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772922.984712:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772922.984721:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772922.984759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772923.048707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772923.112727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772923.112727:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772923.112761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772923.240737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772923.240740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772923.240740:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772923.240775:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772923.304724:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772923.304742:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772923.304752:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772923.304780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772923.304791:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772923.304796:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772923.432712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772923.496737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772923.560737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772923.624738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772923.624763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772923.624803:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772923.624805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772923.688730:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772923.688761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772923.688895:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772923.816773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772923.816778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772923.880782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772923.944778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772924.008752:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772924.008752:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772924.008755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772924.072742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772924.136719:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772924.136738:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772924.136743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772924.264760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772924.264773:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772924.264781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772924.264793:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772924.328771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772924.328771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772924.328780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772924.328782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772924.328816:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772924.328827:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772924.456711:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772924.520756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772924.584737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:1.0:1790772924.584934:0:2386:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:1.0:1790772924.584981:0:2386:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1790772924.584999:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772924.585048:0:2386:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:1.0:1790772924.585063:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772924.585071:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790772924.585078:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772921/real 1790772924] req@ffff9c4efbcc0a80 x1877759534560896/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790772924.585079:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772924.585092:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772924.585107:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790772924.585122:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534560896/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790772924.585123:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790772924.585139:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:0.0:1790772924.585144:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772924.585160:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772924.585167:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534560896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772924.585178:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772921/real 1790772924] req@ffff9c4efbcc0700 x1877759534560768/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772924.585194:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534560768/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772924.585206:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772924.585210:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772924.585214:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534560768:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772924.585223:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772921/real 1790772924] req@ffff9c4efbcc2680 x1877759534560640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772924.585237:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534560640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772924.585248:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772924.585252:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772924.585256:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534560640:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772924.585263:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772921/real 1790772924] req@ffff9c4efbcc2300 x1877759534560512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772924.585277:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534560512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772924.585298:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772924.585302:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772924.585306:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534560512:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772924.585313:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772921/real 1790772924] req@ffff9c4efbcc1880 x1877759534560384/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772924.585326:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534560384/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772924.585336:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772924.585340:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772924.585344:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534560384:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772924.585351:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772921/real 1790772924] req@ffff9c4efbcc1500 x1877759534560256/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772924.585363:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534560256/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772924.585374:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772924.585378:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772924.585382:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534560256:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772924.585389:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772921/real 1790772924] req@ffff9c4efbcc1f80 x1877759534560000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772924.585403:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534560000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772924.585423:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772924.585428:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772924.585432:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534560000:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772924.585440:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772921/real 1790772924] req@ffff9c4efbcc1c00 x1877759534560128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772924.585453:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534560128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772924.585463:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772924.585467:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772924.585471:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534560128:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772924.585479:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772921/real 1790772924] req@ffff9c4efbcc2a00 x1877759534559872/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772924.585493:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534559872/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772924.585503:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772924.585508:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772924.585512:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534559872:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772924.648762:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772924.648763:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772924.648785:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772924.648785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772924.712763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772924.712865:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772924.712884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772924.840726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772924.840726:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772924.904759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772924.968748:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772925.032740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772925.032740:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772925.032777:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772925.096699:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772925.160729:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772925.160729:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772925.160763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772925.288721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772925.288725:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772925.288741:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772925.288775:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772925.352739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772925.352739:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772925.352740:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772925.352742:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772925.352781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772925.352805:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772925.480722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772925.544734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772925.608721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772925.672753:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772925.672757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772925.672761:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772925.672765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772925.736762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772925.736850:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772925.736882:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772925.864739:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772925.864747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772925.928749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772925.992714:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772926.056787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772926.056787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772926.056788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772926.120721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772926.184705:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772926.184725:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772926.184766:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772926.312746:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772926.312746:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772926.312746:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772926.312785:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772926.376721:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772926.376721:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772926.376739:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772926.376776:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772926.376785:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772926.376803:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772926.504708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772926.568722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772926.632731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772926.632766:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772926.632775:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772926.632780:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772926.632791:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1922 00000100:00080000:3.0:1790772926.632803:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772926.632818:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772926.632860:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534561024/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772926.632888:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772926.632891:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772926.632893:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772926.632897:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1922 00000100:00080000:3.0:1790772926.632902:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772926.632909:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772926.632923:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772926.632937:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534561024:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772926.632944:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534561152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772926.632959:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772926.632965:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772926.632967:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772926.632973:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1922 00000100:00080000:3.0:1790772926.632977:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772926.632984:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772926.632990:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772926.633008:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534561280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772926.633017:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772926.633020:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772926.633030:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790772926.633031:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772926.633041:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1922 00000100:00080000:3.0:1790772926.633050:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772926.633068:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772926.633077:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772926.633093:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772926/real 1790772926] req@ffff9c4ef73c2d80 x1877759534561024/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772926.633100:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534561408/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772926.633125:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772926.633129:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534561024/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772926.633132:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772926.633136:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772926.633144:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1922 00000100:00080000:3.0:1790772926.633149:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772926.633155:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772926.633167:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772926.633188:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534561536/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772926.633189:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772926.633207:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772926.633219:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772926.633224:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534561024:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772926.633231:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772926.633236:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772926.633255:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1922 00000100:00080000:3.0:1790772926.633260:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772926.633278:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772926.633279:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534561152:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772926.633293:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772926.633316:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534561664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790772926.633330:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772926.633337:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772926.633345:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772926.633347:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772926.633361:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1922 00000100:00100000:0.0:1790772926.633365:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772926/real 1790772926] req@ffff9c4ef73c1500 x1877759534561152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772926.633370:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772926.633384:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772926.633399:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772926.633405:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534561152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772926.633420:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534561792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772926.633431:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772926.633435:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772926.633445:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790772926.633446:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772926.633451:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790772926.633459:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534561152:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772926.633464:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1922 00000100:00080000:3.0:1790772926.633470:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772926.633483:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772926.633504:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772926.633505:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534561280:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772926.633531:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534561920/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772926.633547:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790772926.633551:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772926.633560:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1790772926.633578:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772926/real 1790772926] req@ffff9c4ef73c0a80 x1877759534561280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772926.633594:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534561280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772926.633605:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772926.633609:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772926.633613:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534561280:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772926.633623:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534561408:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772926.633648:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772926.633696:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772926/real 1790772926] req@ffff9c4ef73c3800 x1877759534561408/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772926.633708:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534561408/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772926.633722:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772926.633728:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772926.633733:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790772926.633734:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534561408:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772926.633745:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1922 00000100:00080000:3.0:1790772926.633758:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772926.633763:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534561536:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772926.633792:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772926.633803:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:0.0:1790772926.633818:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772926.633826:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534562048/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772926.633850:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772926/real 1790772926] req@ffff9c4ef73c2a00 x1877759534561536/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772926.633868:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534561536/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772926.633880:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772926.633884:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772926.633888:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534561536:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772926.633928:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534561664:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772926.633961:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772926.633974:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772926/real 1790772926] req@ffff9c4ef73c0700 x1877759534561664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772926.633987:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534561664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772926.634006:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772926.634009:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772926.634013:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534561664:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772926.634019:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534561792:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772926.634040:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772926.634052:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772926/real 1790772926] req@ffff9c4ef73c2300 x1877759534561792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772926.634064:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534561792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772926.634073:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772926.634077:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772926.634081:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534561792:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772926.634088:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534561920:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772926.634108:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772926.634120:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772926/real 1790772926] req@ffff9c4ef73c0000 x1877759534561920/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772926.634132:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534561920/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772926.634150:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772926.634154:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772926.634158:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534561920:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772926.634165:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534562048:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772926.634185:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772926.634197:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772926/real 1790772926] req@ffff9c4ef73c1c00 x1877759534562048/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772926.634208:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534562048/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772926.634218:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772926.634222:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772926.634226:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534562048:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772926.696742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772926.696742:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772926.696808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772926.696819:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772926.760754:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790772926.760760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772926.760894:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772926.888712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772926.888716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772926.952727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772927.016821:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772927.080749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772927.080754:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772927.080754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772927.144707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772927.208733:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772927.208733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772927.208768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772927.336734:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772927.336739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772927.336766:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772927.336767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772927.400743:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772927.400745:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772927.400750:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772927.400786:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772927.400819:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772927.400842:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772927.528708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772927.592708:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772927.656717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772927.720732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772927.720741:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772927.720766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772927.720772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772927.784719:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772927.784746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772927.784810:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772927.912735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772927.912736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772927.976704:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772928.040723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772928.104729:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772928.104730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772928.104730:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772928.168722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772928.232785:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772928.232785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772928.232785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772928.360744:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772928.360795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772928.360807:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772928.360811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772928.424747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772928.424780:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772928.424795:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772928.424818:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772928.424818:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772928.424856:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772928.552716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772928.616737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772928.680720:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772928.744755:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772928.744781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772928.744784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772928.744787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772928.808754:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772928.808787:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772928.808879:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772928.936702:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772928.936711:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772929.000724:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772929.064786:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772929.128730:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772929.128730:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772929.128731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772929.192713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772929.256748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772929.256748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772929.256751:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772929.384711:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772929.384736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772929.384740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772929.384744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772929.448725:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772929.448732:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772929.448734:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772929.448747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772929.448768:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772929.448776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772929.576709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772929.640711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772929.704733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772929.768743:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772929.768743:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772929.768743:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772929.768773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772929.832717:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772929.832717:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772929.832787:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772929.960710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772929.960731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772930.024715:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772930.088751:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772930.152729:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772930.152729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772930.152781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772930.216724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772930.280723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772930.280726:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772930.280743:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772930.408776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772930.408779:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772930.408781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772930.408786:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772930.472723:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772930.472753:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772930.472775:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772930.472798:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772930.472808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772930.472812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772930.600705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772930.664741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772930.728772:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772930.792765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772930.792765:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772930.792766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772930.792773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772930.856746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772930.856881:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772930.856930:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772930.984728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772930.984730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772931.048715:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772931.112740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772931.176748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772931.176760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772931.176763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772931.240765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772931.304759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772931.304774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772931.304775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772931.432714:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772931.432737:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772931.432747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772931.432758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772931.496747:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772931.496786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772931.496787:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772931.496825:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772931.496828:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772931.496852:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772931.624714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772931.688717:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772931.752749:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772931.752754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772931.752759:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772931.752767:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772931.752778:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1927 00000100:00080000:3.0:1790772931.752788:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772931.752799:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772931.752842:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534562176/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772931.752875:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772931.752878:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772931.752880:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772931.752884:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1927 00000100:00080000:3.0:1790772931.752889:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772931.752894:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772931.752903:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772931.752916:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534562176:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790772931.752920:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534562304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772931.752935:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772931.752938:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772931.752942:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772931.752948:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1927 00000100:00080000:3.0:1790772931.752950:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772931.752954:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772931.752959:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772931.752972:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534562432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772931.752979:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772931.752982:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772931.752983:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772931.752986:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1927 00000100:00080000:3.0:1790772931.752989:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772931.752994:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772931.752998:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:0.0:1790772931.753012:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772931.753012:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534562560/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772931.753026:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772931.753028:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772931.753030:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772931.753036:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1927 00000100:00080000:3.0:1790772931.753039:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772931.753045:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772931.753051:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772931.753069:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772931/real 1790772931] req@ffff9c4ef73c0e00 x1877759534562176/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772931.753074:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534562688/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772931.753101:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534562176/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772931.753101:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772931.753120:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772931.753125:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772931.753126:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772931.753137:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772931.753142:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1927 00000100:00100000:0.0:1790772931.753147:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534562176:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772931.753155:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772931.753166:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772931.753171:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772931.753179:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534562304:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772931.753185:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534562816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772931.753198:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772931.753201:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772931.753203:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790772931.753207:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772931.753215:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1927 00000100:00080000:3.0:1790772931.753221:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772931.753225:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772931/real 1790772931] req@ffff9c4ef73c1c00 x1877759534562304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772931.753229:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772931.753247:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772931.753249:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534562304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772931.753272:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790772931.753272:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534562944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772931.753283:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772931.753295:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534562304:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772931.753314:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534562432:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772931.753318:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772931.753324:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772931.753328:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790772931.753335:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772931.753337:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1927 00000100:00080000:3.0:1790772931.753346:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772931.753347:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772931/real 1790772931] req@ffff9c4ef73c0000 x1877759534562432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772931.753364:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772931.753366:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534562432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772931.753375:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790772931.753414:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790772931.753417:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534563072/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772931.753426:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772931.753441:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534562432:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772931.753452:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772931.753457:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534562560:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790772931.753460:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772931.753465:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772931.753473:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1927 00000100:00080000:3.0:1790772931.753479:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790772931.753485:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772931.753491:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772931.753501:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772931/real 1790772931] req@ffff9c4ef73c2300 x1877759534562560/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790772931.753502:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772931.753523:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534562560/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772931.753532:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534563200/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772931.753534:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772931.753545:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772931.753556:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534562560:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772931.753570:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534562688:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772931.753589:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772931.753600:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772931/real 1790772931] req@ffff9c4ef73c0700 x1877759534562688/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772931.753608:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534562688/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772931.753614:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772931.753617:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772931.753620:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534562688:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772931.753636:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534562816:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772931.753676:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772931.753690:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772931/real 1790772931] req@ffff9c4ef73c2a00 x1877759534562816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772931.753700:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534562816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772931.753712:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772931.753714:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772931.753717:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534562816:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772931.753722:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534562944:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772931.753738:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772931.753746:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772931/real 1790772931] req@ffff9c4ef73c3800 x1877759534562944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772931.753754:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534562944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772931.753760:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772931.753762:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772931.753764:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534562944:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772931.753768:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534563072:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772931.753779:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772931.753786:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772931/real 1790772931] req@ffff9c4ef73c0a80 x1877759534563072/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772931.753794:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534563072/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772931.753802:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772931.753804:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772931.753806:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534563072:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772931.753810:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534563200:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772931.753821:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772931.753828:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772931/real 1790772931] req@ffff9c4ef73c1500 x1877759534563200/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772931.753835:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534563200/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772931.753842:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772931.753845:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772931.753849:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534563200:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772931.816734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772931.816773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772931.816779:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772931.816790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772931.880724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772931.880786:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772931.880913:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772932.008740:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772932.008740:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772932.072745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772932.136747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772932.200730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772932.200730:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772932.200741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772932.264755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772932.328725:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772932.328775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772932.328779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772932.456740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772932.456743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772932.456750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772932.456759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772932.520724:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772932.520744:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772932.520769:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772932.520771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772932.520787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772932.520822:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772932.648716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772932.712742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772932.776716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772932.840733:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772932.840797:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772932.840806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772932.840808:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772932.904728:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772932.904759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772932.904831:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772933.032728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772933.032747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772933.096724:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772933.160722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772933.224738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772933.224738:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772933.224738:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772933.288723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772933.352748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772933.352774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772933.352783:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772933.480772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772933.480776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772933.480780:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772933.480782:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772933.544719:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772933.544720:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772933.544730:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772933.544738:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772933.544758:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772933.544768:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772933.672709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772933.736717:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772933.800706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772933.864738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772933.864738:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772933.864738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772933.864739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772933.928715:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772933.928720:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772933.928802:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772934.056745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772934.056745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772934.120735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772934.184794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772934.248731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772934.248731:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772934.248764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772934.312726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772934.376711:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772934.376790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772934.376791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772934.504746:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772934.504777:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772934.504783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772934.504783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772934.568736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772934.568786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772934.568788:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772934.568789:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772934.568821:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772934.568838:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772934.696712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772934.760750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772934.824714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772934.888763:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772934.888763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772934.888789:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772934.888797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772934.952735:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772934.952736:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772934.952834:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772935.080725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772935.080726:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772935.144742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772935.208723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772935.272724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772935.272766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772935.272777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772935.336711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772935.400766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772935.400767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772935.400767:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772935.528722:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772935.528740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772935.528744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772935.528764:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772935.592728:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772935.592735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772935.592760:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772935.592780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772935.592782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772935.592806:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772935.720723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772935.784734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772935.848720:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772935.912759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772935.912760:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772935.912792:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772935.912793:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772935.976731:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772935.976732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772935.976892:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772936.104734:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772936.104740:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772936.168771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772936.232731:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772936.296748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772936.296748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772936.296755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772936.360736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772936.424744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772936.424744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772936.424770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772936.552758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772936.552759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772936.552794:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772936.552795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772936.616751:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772936.616762:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772936.616762:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772936.616763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772936.616802:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772936.616817:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772936.744724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772936.808751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790772936.872758:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772936.872766:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1790772936.872773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772936.872773:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772936.872791:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1932 00000100:00080000:1.0:1790772936.872800:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772936.872811:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772936.872848:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534563328/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772936.872873:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772936.872877:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772936.872883:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772936.872887:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1932 00000100:00080000:1.0:1790772936.872890:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772936.872894:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772936.872902:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772936.872913:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534563456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772936.872920:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772936.872923:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772936.872925:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772936.872928:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1932 00000100:00080000:1.0:1790772936.872931:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772936.872935:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772936.872938:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772936.872950:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534563584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772936.872951:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534563328:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772936.872965:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772936.872968:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772936.872969:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772936.872972:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1932 00000100:00080000:1.0:1790772936.872975:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772936.872978:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772936.872982:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772936.872992:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534563712/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772936.872999:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772936.873001:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772936.873003:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772936.873006:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1932 00000100:00080000:1.0:1790772936.873011:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772936.873013:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772936.873017:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:2.0:1790772936.873020:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772936.873028:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534563840/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772936.873035:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772936.873038:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772936.873040:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772936.873043:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1932 00000100:00080000:1.0:1790772936.873045:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772936.873048:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772936.873054:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772936.873070:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534563968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772936.873072:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772936/real 1790772936] req@ffff9c4efac4fb80 x1877759534563328/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772936.873099:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772936.873104:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534563328/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772936.873105:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772936.873110:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772936.873119:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772936.873121:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1932 00000100:00080000:2.0:1790772936.873126:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772936.873129:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772936.873131:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534563328:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772936.873139:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772936.873141:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534563456:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790772936.873147:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772936.873162:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534564096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772936.873165:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772936.873177:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772936/real 1790772936] req@ffff9c4efac4e300 x1877759534563456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772936.873179:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772936.873189:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772936.873193:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772936.873196:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534563456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772936.873201:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1932 00000100:00080000:1.0:1790772936.873210:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772936.873215:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772936.873217:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772936.873222:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790772936.873224:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772936.873228:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534563456:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772936.873239:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534564224/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772936.873247:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772936.873249:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772936.873252:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772936.873253:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534563584:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772936.873259:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1932 00000100:00080000:1.0:1790772936.873262:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772936.873265:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772936.873268:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772936.873280:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534564352/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772936.873288:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772936.873321:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772936/real 1790772936] req@ffff9c4efac4ea00 x1877759534563584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772936.873330:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534563584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772936.873337:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772936.873339:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772936.873341:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534563584:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772936.873346:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534563712:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772936.873359:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772936.873366:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772936/real 1790772936] req@ffff9c4efac4e680 x1877759534563712/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772936.873374:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534563712/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772936.873380:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772936.873382:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772936.873384:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534563712:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772936.873388:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534563840:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772936.873398:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772936.873405:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772936/real 1790772936] req@ffff9c4efac4f480 x1877759534563840/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772936.873413:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534563840/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772936.873419:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772936.873421:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772936.873423:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534563840:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772936.873427:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534563968:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772936.873437:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772936.873446:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772936/real 1790772936] req@ffff9c4efac4f800 x1877759534563968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772936.873458:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534563968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772936.873474:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772936.873478:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772936.873482:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534563968:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772936.873488:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534564096:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772936.873508:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772936.873518:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772936/real 1790772936] req@ffff9c4efac4dc00 x1877759534564096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772936.873526:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534564096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772936.873541:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772936.873543:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772936.873545:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534564096:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772936.873550:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534564224:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772936.873562:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772936.873569:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772936/real 1790772936] req@ffff9c4efac4ed80 x1877759534564224/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772936.873577:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534564224/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772936.873584:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772936.873586:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772936.873588:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534564224:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772936.873604:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534564352:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772936.873614:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772936.873621:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772936/real 1790772936] req@ffff9c4efac4c700 x1877759534564352/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772936.873628:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534564352/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772936.873634:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772936.873636:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772936.873639:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534564352:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772936.936770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772936.936770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772936.936770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772936.936770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772937.000727:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772937.000746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772937.000794:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772937.128721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772937.128723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772937.192718:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772937.256722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772937.320764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772937.320764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772937.320764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772937.384741:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772937.448741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772937.448741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772937.448742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772937.576738:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772937.576738:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772937.576738:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772937.576738:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772937.640728:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772937.640728:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772937.640730:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772937.640771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772937.640783:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772937.640785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772937.768700:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772937.832719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772937.896707:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772937.960774:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772937.960774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772937.960798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772937.960801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772938.024740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772938.024740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772938.024828:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772938.152776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772938.152782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772938.216743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772938.280710:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772938.344742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772938.344768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772938.344768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772938.408759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772938.472715:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772938.472721:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772938.472741:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772938.600796:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772938.600805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772938.600808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772938.600812:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772938.664725:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772938.664758:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772938.664759:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772938.664759:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772938.664768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772938.664804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772938.792709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772938.856747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772938.920713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772938.984733:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772938.984733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772938.984733:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772938.984768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772939.048714:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772939.048733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772939.048810:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772939.176716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772939.176717:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772939.240744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772939.304730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772939.368759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772939.368760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772939.368760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772939.432707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772939.496718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772939.496722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772939.496730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772939.624721:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772939.624728:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772939.624729:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772939.624733:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772939.688730:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772939.688732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772939.688733:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772939.688749:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772939.688769:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772939.688782:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772939.816715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772939.880716:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772939.944711:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772940.008711:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772940.008713:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772940.008734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772940.008757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772940.072711:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772940.072777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772940.072857:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772940.200708:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772940.200712:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772940.264716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772940.328744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772940.392720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772940.392743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772940.392750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772940.456736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772940.520731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772940.520732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772940.520738:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772940.648751:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772940.648752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772940.648752:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772940.648753:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772940.712746:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772940.712746:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772940.712799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772940.712801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772940.712830:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772940.712839:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772940.840705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772940.904769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772940.968720:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772941.032745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772941.032769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772941.032776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772941.032793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772941.096776:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772941.096779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772941.096846:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772941.224718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772941.224728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772941.288735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772941.352828:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772941.416707:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772941.416709:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772941.416730:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772941.480738:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772941.544749:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772941.544749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772941.544749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772941.672726:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772941.672727:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772941.672727:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772941.672775:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772941.736731:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772941.736731:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772941.736771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772941.736771:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772941.736802:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772941.736817:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772941.864712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772941.928726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790772941.992735:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772941.992739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772941.992744:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772941.992750:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772941.992767:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1937 00000100:00080000:1.0:1790772941.992776:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772941.992786:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772941.992826:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534564480/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772941.992855:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772941.992858:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772941.992860:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772941.992864:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1937 00000100:00080000:1.0:1790772941.992867:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772941.992880:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772941.992892:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772941.992901:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534564480:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772941.992907:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534564608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772941.992921:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772941.992925:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772941.992928:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772941.992934:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1937 00000100:00080000:1.0:1790772941.992939:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772941.992965:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772941.992978:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790772941.992992:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772941.993006:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534564736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772941.993023:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772941.993028:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772941.993031:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772941.993037:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1937 00000100:00080000:1.0:1790772941.993062:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772941.993074:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772941.993078:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772941/real 1790772941] req@ffff9c4efac4df80 x1877759534564480/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772941.993083:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772941.993106:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534564864/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772941.993123:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772941.993130:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1790772941.993135:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534564480/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772941.993136:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772941.993150:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1937 00000100:00080000:1.0:1790772941.993158:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772941.993169:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772941.993224:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790772941.993236:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:2.0:1790772941.993238:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772941.993264:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534564480:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772941.993269:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534564992/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772941.993277:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772941.993280:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772941.993282:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772941.993285:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1937 00000100:00080000:1.0:1790772941.993288:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772941.993293:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772941.993297:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534564608:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790772941.993299:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772941.993312:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534565120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772941.993321:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772941.993323:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772941.993325:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790772941.993329:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772941.993330:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1937 00000100:00080000:1.0:1790772941.993336:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772941.993341:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772941.993345:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772941.993346:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772941/real 1790772941] req@ffff9c4efac4c700 x1877759534564608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772941.993355:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534565248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772941.993363:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534564608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772941.993365:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772941.993372:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772941.993374:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772941.993382:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1937 00000100:00080000:1.0:1790772941.993395:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772941.993400:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772941.993406:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:2.0:1790772941.993408:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772941.993415:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772941.993423:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534564608:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772941.993424:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534565376/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772941.993432:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534564736:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772941.993445:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772941.993449:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772941.993453:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772941.993480:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1937 00000800:00000100:2.0:1790772941.993485:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772941.993490:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772941.993505:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772941.993515:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772941.993517:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772941/real 1790772941] req@ffff9c4efac4ed80 x1877759534564736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772941.993543:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534565504/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772941.993548:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534564736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772941.993605:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772941.993612:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772941.993633:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534564736:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772941.993639:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534564864:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772941.993654:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772941.993662:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772941/real 1790772941] req@ffff9c4efac4dc00 x1877759534564864/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772941.993734:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534564864/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772941.993755:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772941.993763:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772941.993771:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534564864:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772941.993785:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534564992:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772941.993838:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772941.993852:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772941/real 1790772941] req@ffff9c4efac4f800 x1877759534564992/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772941.993867:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534564992/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772941.993875:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772941.993877:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772941.993880:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534564992:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772941.993898:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534565120:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772941.993914:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772941.993922:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772941/real 1790772941] req@ffff9c4efac4f480 x1877759534565120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772941.993931:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534565120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772941.993944:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772941.993965:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772941.993970:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534565120:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772941.993978:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534565248:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772941.994001:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772941.994013:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772941/real 1790772941] req@ffff9c4efac4e680 x1877759534565248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772941.994029:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534565248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772941.994041:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772941.994046:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772941.994050:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534565248:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772941.994059:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534565376:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772941.994092:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772941.994114:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772941/real 1790772941] req@ffff9c4efac4ea00 x1877759534565376/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772941.994140:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534565376/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772941.994163:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772941.994171:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772941.994180:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534565376:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772941.994222:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534565504:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772941.994264:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772941.994277:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772941/real 1790772941] req@ffff9c4efac4e300 x1877759534565504/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772941.994286:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534565504/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772941.994293:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772941.994296:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772941.994299:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534565504:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772942.056836:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772942.056852:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772942.056853:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772942.056889:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772942.120709:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772942.120734:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772942.120823:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772942.248768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772942.248836:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772942.312810:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772942.376857:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772942.440730:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772942.440744:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772942.440848:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772942.504751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772942.568799:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772942.568812:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772942.568837:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772942.696806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772942.696807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772942.696809:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772942.696821:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772942.760724:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772942.760738:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772942.760771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772942.760776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772942.760778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772942.760797:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772942.888713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772942.952725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772943.016784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772943.080720:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772943.080721:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772943.080743:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772943.080803:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772943.144729:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772943.144790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772943.144846:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772943.272747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772943.272747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772943.336735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772943.400737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772943.464697:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772943.464702:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772943.464755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772943.528716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772943.592743:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772943.592746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772943.592780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772943.720792:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772943.720792:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772943.720794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772943.720798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772943.784719:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772943.784725:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772943.784766:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772943.784777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772943.784781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772943.784809:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772943.912741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772943.976703:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772944.040718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772944.104751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772944.104751:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772944.104770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772944.104790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772944.168721:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772944.168731:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772944.168884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772944.296721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772944.296723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772944.360768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772944.424735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772944.488709:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772944.488790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772944.488791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772944.552720:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772944.616767:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772944.616767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772944.616787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772944.744758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772944.744758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772944.744760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772944.744760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772944.808749:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772944.808753:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772944.808759:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772944.808761:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772944.808803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772944.808808:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772944.936707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772945.000699:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772945.064714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772945.128740:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772945.128741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772945.128745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772945.128777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772945.192722:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772945.192724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772945.192836:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772945.320717:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772945.320723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772945.384742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772945.448716:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772945.512720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772945.512724:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772945.512752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772945.576701:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772945.640710:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772945.640724:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772945.640784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772945.768743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772945.768743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772945.768743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772945.768764:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772945.832705:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772945.832738:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772945.832738:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772945.832751:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772945.832779:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772945.832787:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772945.960719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772946.024707:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790772946.088726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772946.152742:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772946.152744:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772946.152746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772946.152746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772946.216727:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772946.216729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772946.216754:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772946.344720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772946.344723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772946.408742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772946.472719:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772946.536731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772946.536731:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772946.536756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772946.600745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772946.664749:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772946.664749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772946.664757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772946.792723:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772946.792730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772946.792730:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772946.792743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772946.856744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772946.856744:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772946.856746:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772946.856763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772946.856794:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772946.856796:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772946.984705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772947.048744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772947.112746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790772947.112760:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772947.112768:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790772947.112774:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772947.112785:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1943 00000100:00080000:2.0:1790772947.112796:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772947.112807:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772947.112844:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534565632/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772947.112867:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772947.112871:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772947.112874:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772947.112879:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1943 00000100:00080000:2.0:1790772947.112884:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772947.112887:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534565632:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772947.112893:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772947.112906:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772947.112923:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534565760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772947.112935:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772947.112940:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772947.112942:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790772947.112948:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772947.112951:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1943 00000100:00080000:2.0:1790772947.112958:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772947.112965:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772947.112971:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772947.112987:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772947/real 1790772947] req@ffff9c4efbcc3100 x1877759534565632/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772947.112990:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534565888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772947.113017:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772947.113019:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534565632/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772947.113024:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772947.113030:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772947.113034:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772947.113039:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1943 00000100:00080000:0.0:1790772947.113041:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772947.113048:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534565632:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790772947.113048:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772947.113060:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772947.113067:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772947.113077:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534565760:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772947.113085:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534566016/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772947.113111:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772947.113116:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772947.113119:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790772947.113125:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790772947.113127:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1943 00000100:00080000:2.0:1790772947.113134:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772947.113141:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772947.113145:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772947/real 1790772947] req@ffff9c4efbcc0380 x1877759534565760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790772947.113149:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772947.113171:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534566144/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772947.113180:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534565760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772947.113184:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772947.113195:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790772947.113198:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772947.113204:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1943 00000100:00080000:0.0:1790772947.113206:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772947.113209:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772947.113212:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772947.113217:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534565760:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772947.113222:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772947.113227:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534565888:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790772947.113232:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790772947.113250:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772947.113259:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534566272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772947.113262:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772947/real 1790772947] req@ffff9c4efbcc2a00 x1877759534565888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772947.113293:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534565888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772947.113295:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772947.113306:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790772947.113311:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772947.113313:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772947.113321:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772947.113322:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1943 00000100:00100000:0.0:1790772947.113325:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534565888:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772947.113331:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772947.113339:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772947.113344:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534566016:192.168.204.150@tcp:8:kworker.0 00000008:00100000:2.0:1790772947.113347:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790772947.113364:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772947.113367:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534566400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772947.113376:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772947/real 1790772947] req@ffff9c4efbcc1c00 x1877759534566016/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772947.113389:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772947.113394:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534566016/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772947.113396:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772947.113401:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772947.113412:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1943 00000100:00080000:0.0:1790772947.113418:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772947.113419:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772947.113425:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790772947.113430:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772947.113431:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534566016:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772947.113441:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534566144:192.168.204.150@tcp:8:kworker.0 00000008:00100000:2.0:1790772947.113443:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:0.0:1790772947.113459:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772947.113462:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534566528/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772947.113470:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772947/real 1790772947] req@ffff9c4efbcc1f80 x1877759534566144/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772947.113482:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772947.113487:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534566144/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772947.113489:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790772947.113495:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772947.113500:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772947.113504:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1943 00000100:00080000:0.0:1790772947.113507:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772947.113512:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534566144:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790772947.113513:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772947.113520:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534566272:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790772947.113525:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790772947.113534:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:0.0:1790772947.113539:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772947.113549:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534566656/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772947.113558:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772947/real 1790772947] req@ffff9c4efbcc1500 x1877759534566272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772947.113570:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534566272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772947.113583:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772947.113586:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772947.113589:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534566272:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772947.113601:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534566400:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772947.113613:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772947.113620:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772947/real 1790772947] req@ffff9c4efbcc1880 x1877759534566400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772947.113629:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534566400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772947.113635:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772947.113637:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772947.113640:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534566400:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772947.113643:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534566528:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772947.113672:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772947.113679:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772947/real 1790772947] req@ffff9c4efbcc2300 x1877759534566528/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772947.113687:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534566528/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772947.113695:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772947.113697:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772947.113700:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534566528:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772947.113704:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534566656:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772947.113714:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772947.113721:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772947/real 1790772947] req@ffff9c4efbcc2680 x1877759534566656/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772947.113729:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534566656/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772947.113735:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772947.113737:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772947.113739:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534566656:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772947.176704:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772947.176729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772947.176733:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772947.176748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772947.240710:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772947.240736:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772947.240797:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772947.368735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772947.368735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772947.432758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772947.496731:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772947.560810:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772947.560811:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772947.560816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772947.624724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772947.688811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772947.688813:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772947.688814:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772947.816731:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772947.816759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772947.816767:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772947.816823:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772947.880749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772947.880750:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772947.880784:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772947.880793:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772947.880806:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772947.880812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772948.008730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772948.072740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772948.136739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772948.200742:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772948.200742:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772948.200766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772948.200790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772948.264750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772948.264777:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772948.264780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772948.392821:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772948.392821:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772948.456775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772948.520746:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772948.584756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772948.584757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772948.584780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772948.648720:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772948.712742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772948.712745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772948.712749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772948.840731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772948.840766:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772948.840773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772948.840776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772948.904734:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772948.904748:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772948.904780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772948.904780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772948.904811:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772948.904824:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772949.032722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772949.096730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772949.160724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772949.224764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772949.224764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772949.224764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772949.224764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772949.288744:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772949.288778:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772949.288800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772949.416719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772949.416732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772949.480735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772949.544733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772949.608747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772949.608747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772949.608747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772949.672734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772949.736757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772949.736757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772949.736758:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772949.864737:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772949.864737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772949.864737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772949.864738:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772949.928737:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772949.928737:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772949.928755:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772949.928829:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772949.928833:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772949.928835:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772950.056735:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772950.120740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772950.184730:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772950.248729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772950.248730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772950.248747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772950.248753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772950.312738:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772950.312741:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772950.312796:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772950.440736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772950.440742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772950.504714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772950.568725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772950.632760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772950.632760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772950.632779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772950.696794:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772950.760782:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772950.760782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772950.760805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772950.888739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772950.888739:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772950.888740:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772950.888746:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772950.952724:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772950.952754:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772950.952774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772950.952774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772950.952784:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772950.952809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772951.080719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772951.144725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790772951.208710:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772951.272736:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772951.272788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772951.272806:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772951.272807:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772951.336745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772951.336749:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772951.336814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772951.464733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772951.464743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772951.528752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772951.592730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772951.656728:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772951.656738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772951.656744:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772951.720733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772951.784715:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772951.784745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772951.784747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772951.912729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772951.912729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772951.912732:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772951.912732:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772951.976767:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772951.976767:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772951.976776:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772951.976776:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772951.976832:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772951.976833:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772952.104723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772952.168742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790772952.232738:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772952.232750:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1790772952.232750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772952.232755:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772952.232767:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1948 00000100:00080000:1.0:1790772952.232779:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772952.232792:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772952.232840:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534566784/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772952.232872:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772952.232877:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772952.232879:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772952.232884:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1948 00000100:00080000:1.0:1790772952.232888:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772952.232894:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772952.232907:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772952.232925:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534566784:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772952.232926:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534566912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772952.232947:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772952.232950:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772952.232952:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772952.232955:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1948 00000100:00080000:1.0:1790772952.232959:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772952.232963:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772952.232967:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772952.232977:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534567040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772952.232984:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772952.232987:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772952.232989:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772952.233005:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1948 00000100:00080000:1.0:1790772952.233013:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772952.233017:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772952.233022:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:0.0:1790772952.233029:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772952.233034:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534567168/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772952.233042:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772952.233045:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772952.233046:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772952.233050:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1948 00000100:00080000:1.0:1790772952.233053:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772952.233060:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772952.233063:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772952.233071:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534567296/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772952.233078:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772952.233082:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772952.233084:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790772952.233089:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772952/real 1790772952] req@ffff9c4efac4fb80 x1877759534566784/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772952.233089:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1948 00000100:00080000:1.0:1790772952.233094:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772952.233099:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772952.233104:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772952.233115:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534567424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772952.233126:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534566784/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772952.233130:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772952.233138:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772952.233141:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772952.233148:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1948 00000100:00080000:1.0:1790772952.233151:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772952.233153:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772952.233154:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772952.233163:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790772952.233166:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790772952.233173:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534567552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772952.233175:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534566784:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772952.233189:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772952.233193:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772952.233194:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772952.233198:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1948 00000100:00080000:1.0:1790772952.233201:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772952.233204:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772952.233207:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772952.233215:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534566912:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772952.233216:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534567680/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772952.233236:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772952.233239:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772952.233241:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772952.233245:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1948 00000100:00080000:1.0:1790772952.233249:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772952.233255:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790772952.233257:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772952.233260:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772952.233271:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534567808/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772952.233280:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772952/real 1790772952] req@ffff9c4efac4e300 x1877759534566912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772952.233303:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534566912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772952.233323:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772952.233327:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772952.233331:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534566912:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772952.233338:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534567040:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772952.233359:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772952.233371:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772952/real 1790772952] req@ffff9c4efac4ea00 x1877759534567040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772952.233383:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534567040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772952.233393:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772952.233397:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772952.233401:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534567040:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772952.233408:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534567168:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772952.233427:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772952.233438:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772952/real 1790772952] req@ffff9c4efac4e680 x1877759534567168/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772952.233451:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534567168/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772952.233468:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772952.233472:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772952.233476:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534567168:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772952.233482:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534567296:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772952.233502:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772952.233513:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772952/real 1790772952] req@ffff9c4efac4f480 x1877759534567296/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772952.233525:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534567296/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772952.233535:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772952.233538:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772952.233542:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534567296:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772952.233549:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534567424:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772952.233568:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772952.233579:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772952/real 1790772952] req@ffff9c4efac4f800 x1877759534567424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772952.233591:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534567424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772952.233606:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772952.233609:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772952.233613:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534567424:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772952.233619:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534567552:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772952.233637:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772952.233647:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772952/real 1790772952] req@ffff9c4efac4dc00 x1877759534567552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772952.233685:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534567552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772952.233694:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772952.233698:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772952.233702:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534567552:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772952.233723:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534567680:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772952.233745:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772952.233758:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772952/real 1790772952] req@ffff9c4efac4ed80 x1877759534567680/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772952.233772:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534567680/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772952.233783:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772952.233788:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772952.233792:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534567680:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772952.233798:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534567808:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772952.233817:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772952.233829:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772952/real 1790772952] req@ffff9c4efac4c700 x1877759534567808/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772952.233842:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534567808/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772952.233852:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772952.233855:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772952.233859:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534567808:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772952.296718:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772952.296773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772952.296773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772952.296787:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772952.360745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772952.360774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772952.360804:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772952.488723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772952.488751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772952.552735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772952.616735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772952.680729:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772952.680729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772952.680756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772952.744736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772952.808788:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772952.808789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772952.808793:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772952.936725:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772952.936751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772952.936765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772952.936771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772953.000712:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772953.000741:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772953.000768:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772953.000768:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772953.000782:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772953.000787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772953.128745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772953.192703:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772953.256750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772953.320770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772953.320770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772953.320771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772953.320771:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772953.384737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772953.384781:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772953.384904:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772953.512747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772953.512767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772953.576716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772953.640786:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772953.704739:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772953.704747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772953.704749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772953.768710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772953.832751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772953.832765:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772953.832766:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772953.960737:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772953.960737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772953.960764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772953.960767:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772954.024759:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772954.024771:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772954.024804:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772954.024804:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772954.024839:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772954.024846:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772954.152703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772954.216723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772954.280704:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772954.344729:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772954.344729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772954.344736:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772954.344736:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772954.408737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772954.408764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772954.408898:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772954.536741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772954.536747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772954.600729:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772954.664746:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772954.728733:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772954.728733:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772954.728791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772954.792716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772954.856735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772954.856735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772954.856751:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772954.984754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772954.984757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772954.984768:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772954.984770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772955.048756:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772955.048770:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772955.048778:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772955.048780:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772955.048818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772955.048819:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772955.176703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772955.240711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772955.304709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772955.368746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772955.368747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772955.368750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772955.368765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772955.432752:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772955.432788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772955.432881:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772955.560720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772955.560730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772955.624773:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772955.688715:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772955.752751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772955.752788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772955.752791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772955.816706:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772955.880725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772955.880730:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772955.880769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772956.008753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772956.008754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772956.008756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772956.008757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772956.072717:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772956.072721:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772956.072745:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772956.072781:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772956.072786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772956.072789:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772956.200713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772956.264725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772956.328719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772956.392797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772956.392800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772956.392801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772956.392802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772956.456732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772956.456750:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772956.456837:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772956.584725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772956.584725:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772956.648717:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772956.712764:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772956.776712:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772956.776716:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772956.776761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772956.840728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772956.904754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772956.904754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772956.904764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772957.032722:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772957.032724:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772957.032776:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772957.032798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772957.096726:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772957.096757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772957.096758:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772957.096769:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772957.096778:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772957.096821:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772957.224717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772957.288734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772957.352726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772957.352762:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772957.352774:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772957.352779:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772957.352793:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1953 00000100:00080000:1.0:1790772957.352809:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772957.352825:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772957.352878:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534567936/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772957.352923:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772957.352928:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772957.352930:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772957.352937:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1953 00000100:00080000:1.0:1790772957.352942:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772957.352949:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772957.352962:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772957.352965:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534567936:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772957.352991:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534568064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772957.353007:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772957.353012:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772957.353014:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772957.353021:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1953 00000100:00080000:1.0:1790772957.353036:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790772957.353042:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772957.353047:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772957.353056:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772957.353076:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534568192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772957.353088:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772957.353092:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772957.353095:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772957.353101:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1953 00000100:00100000:0.0:1790772957.353106:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772957/real 1790772957] req@ffff9c4efac4df80 x1877759534567936/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772957.353109:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772957.353120:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772957.353133:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772957.353148:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534567936/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772957.353156:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534568320/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772957.353163:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772957.353172:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772957.353181:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534567936:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772957.353184:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772957.353190:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772957.353193:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772957.353200:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1953 00000100:00080000:1.0:1790772957.353206:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772957.353207:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534568064:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772957.353219:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772957.353228:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:0.0:1790772957.353237:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772957.353251:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534568448/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772957.353255:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772957/real 1790772957] req@ffff9c4efac4c700 x1877759534568064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772957.353314:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534568064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772957.353319:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772957.353327:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772957.353331:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772957.353332:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772957.353346:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772957.353351:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1953 00000100:00100000:0.0:1790772957.353359:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534568064:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772957.353362:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772957.353377:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772957.353380:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534568192:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790772957.353388:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772957.353413:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534568576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790772957.353414:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772957.353429:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772957/real 1790772957] req@ffff9c4efac4ed80 x1877759534568192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772957.353431:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772957.353452:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772957.353460:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790772957.353462:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534568192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772957.353475:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1953 00000100:00080000:0.0:1790772957.353477:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772957.353484:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772957.353496:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772957.353497:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772957.353505:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534568192:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790772957.353511:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772957.353514:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534568320:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772957.353541:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534568704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790772957.353543:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772957.353555:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772957/real 1790772957] req@ffff9c4efac4dc00 x1877759534568320/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772957.353563:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772957.353573:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534568320/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772957.353573:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772957.353584:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772957.353589:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772957.353597:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772957.353598:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1953 00000100:00100000:0.0:1790772957.353603:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534568320:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790772957.353609:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772957.353621:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772957.353627:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534568448:192.168.204.150@tcp:8:kworker.0 00000008:00100000:1.0:1790772957.353635:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:0.0:1790772957.353647:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772957.353678:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772957/real 1790772957] req@ffff9c4efac4f800 x1877759534568448/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772957.353689:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534568448/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772957.353696:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534568832/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772957.353700:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772957.353710:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772957.353721:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534568448:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772957.353735:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534568576:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772957.353741:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772957.353747:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772957.353750:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790772957.353754:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772957.353764:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1953 00000100:00100000:0.0:1790772957.353768:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772957/real 1790772957] req@ffff9c4efac4f480 x1877759534568576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772957.353776:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772957.353783:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534568576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772957.353793:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790772957.353803:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790772957.353804:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790772957.353809:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772957.353816:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534568576:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772957.353822:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534568704:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772957.353834:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534568960/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790772957.353844:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772957.353857:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772957/real 1790772957] req@ffff9c4efac4e680 x1877759534568704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772957.353882:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534568704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772957.353889:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772957.353906:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772957.353909:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534568704:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772957.353923:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534568832:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772957.353939:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772957.353947:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772957/real 1790772957] req@ffff9c4efac4ea00 x1877759534568832/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772957.353955:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534568832/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772957.353963:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772957.353965:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772957.353968:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534568832:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772957.353972:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534568960:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772957.353983:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772957.353989:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772957/real 1790772957] req@ffff9c4efac4e300 x1877759534568960/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772957.353996:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534568960/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772957.354007:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772957.354009:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772957.354012:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534568960:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772957.416747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772957.416748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772957.416750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772957.416793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772957.480709:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772957.480714:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772957.480875:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772957.608716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772957.608721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772957.672780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772957.736731:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772957.800805:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772957.800806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772957.800806:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772957.864704:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772957.928717:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772957.928723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772957.928751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772958.056752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772958.056752:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772958.056790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772958.056795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772958.120728:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772958.120742:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772958.120745:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772958.120758:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772958.120770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772958.120781:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772958.248708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772958.312728:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772958.376723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772958.440712:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772958.440714:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772958.440738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772958.440740:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772958.504713:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772958.504730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772958.504783:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772958.632724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772958.632764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772958.696733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772958.760728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772958.824738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772958.824738:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772958.824740:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772958.888721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772958.952730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772958.952733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772958.952758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772959.080721:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772959.080724:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772959.080725:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772959.080737:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772959.144712:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772959.144743:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772959.144747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772959.144768:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772959.144785:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772959.144793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772959.272707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772959.336731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772959.400738:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772959.464724:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772959.464727:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772959.464764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772959.464764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772959.528732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772959.528734:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772959.528844:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772959.656763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772959.656763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772959.720743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772959.784727:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772959.848702:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772959.848715:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772959.848729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772959.912712:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772959.976766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772959.976766:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772959.976766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772960.104735:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772960.104735:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772960.104736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772960.104738:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772960.168756:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772960.168756:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772960.168794:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772960.168798:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772960.168819:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772960.168819:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772960.296725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772960.360719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772960.424731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772960.488761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772960.488761:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772960.488763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772960.488764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772960.552772:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772960.552772:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790772960.552844:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772960.680742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772960.680747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772960.744702:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772960.808718:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772960.872722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772960.872732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772960.872759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772960.936754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772961.000721:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772961.000732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772961.000767:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772961.128726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772961.128726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772961.128730:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772961.128730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772961.192728:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772961.192728:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772961.192732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772961.192732:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772961.192763:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772961.192775:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772961.320723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772961.384725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772961.448719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772961.512748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772961.512749:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772961.512749:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772961.512749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772961.576702:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772961.576711:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772961.576829:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772961.704730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772961.704739:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772961.768762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772961.832736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772961.896719:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772961.896719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772961.896719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772961.960734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772962.024784:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772962.024785:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772962.024785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772962.152727:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772962.152727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772962.152731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772962.152732:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772962.216731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772962.216731:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772962.216731:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772962.216739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772962.216769:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772962.216772:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772962.344707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772962.408744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790772962.472724:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772962.472726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772962.472734:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772962.472739:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772962.472747:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1958 00000100:00080000:1.0:1790772962.472756:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772962.472766:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772962.472814:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534569088/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772962.472836:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772962.472839:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772962.472841:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772962.472844:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1958 00000100:00080000:1.0:1790772962.472846:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772962.472850:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772962.472857:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772962.472870:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534569216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772962.472879:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772962.472882:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772962.472883:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772962.472887:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1958 00000100:00080000:1.0:1790772962.472890:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772962.472894:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772962.472899:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772962.472909:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534569088:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772962.472910:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534569344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772962.472932:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772962.472935:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772962.472936:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772962.472939:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1958 00000100:00080000:1.0:1790772962.472942:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772962.472948:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772962.472951:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772962.472960:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534569472/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772962.472966:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772962.472969:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772962.472971:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772962.472975:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1958 00000100:00080000:1.0:1790772962.472992:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772962.472996:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772962.473000:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:0.0:1790772962.473005:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772962.473011:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534569600/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772962.473018:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772962.473021:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772962.473023:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772962.473028:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1958 00000100:00080000:1.0:1790772962.473032:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772962.473036:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772962.473040:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772962.473050:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534569728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772962.473059:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772962/real 1790772962] req@ffff9c4efac4fb80 x1877759534569088/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772962.473059:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772962.473079:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772962.473085:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772962.473105:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1958 00000100:00080000:1.0:1790772962.473110:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772962.473119:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772962.473123:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772962.473124:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534569088/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772962.473139:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534569856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772962.473142:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772962.473158:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772962.473160:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772962.473167:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772962.473172:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790772962.473173:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534569088:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772962.473178:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1958 00000100:00080000:1.0:1790772962.473184:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772962.473189:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772962.473193:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772962.473195:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534569216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772962.473202:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534569984/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772962.473215:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772962.473217:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772962.473219:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772962.473223:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1958 00000100:00080000:1.0:1790772962.473227:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772962.473234:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790772962.473237:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772962.473243:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772962.473255:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534570112/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772962.473261:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772962/real 1790772962] req@ffff9c4efac4e300 x1877759534569216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772962.473294:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534569216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772962.473312:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772962.473315:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772962.473318:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534569216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772962.473342:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534569344:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772962.473358:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772962.473365:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772962/real 1790772962] req@ffff9c4efac4ea00 x1877759534569344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772962.473374:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534569344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772962.473380:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772962.473382:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772962.473384:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534569344:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772962.473388:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534569472:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772962.473400:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772962.473406:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772962/real 1790772962] req@ffff9c4efac4e680 x1877759534569472/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772962.473414:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534569472/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772962.473421:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772962.473424:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772962.473426:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534569472:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772962.473430:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534569600:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772962.473442:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772962.473448:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772962/real 1790772962] req@ffff9c4efac4f480 x1877759534569600/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772962.473456:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534569600/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772962.473463:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772962.473465:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772962.473467:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534569600:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772962.473471:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534569728:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772962.473482:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772962.473489:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772962/real 1790772962] req@ffff9c4efac4f800 x1877759534569728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772962.473496:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534569728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772962.473508:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772962.473510:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772962.473512:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534569728:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772962.473516:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534569856:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772962.473527:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772962.473534:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772962/real 1790772962] req@ffff9c4efac4dc00 x1877759534569856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772962.473547:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534569856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772962.473554:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772962.473556:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772962.473558:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534569856:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772962.473562:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534569984:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772962.473572:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772962.473579:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772962/real 1790772962] req@ffff9c4efac4ed80 x1877759534569984/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772962.473587:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534569984/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772962.473593:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772962.473596:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772962.473598:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534569984:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772962.473602:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534570112:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772962.473613:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772962.473619:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772962/real 1790772962] req@ffff9c4efac4c700 x1877759534570112/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772962.473626:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534570112/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772962.473637:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772962.473640:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772962.473642:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534570112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772962.536732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772962.536772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772962.536812:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772962.536813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772962.600738:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772962.600749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772962.600828:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772962.728732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772962.728749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772962.792711:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772962.856730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772962.920703:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772962.920706:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772962.920726:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772962.984713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772963.048712:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772963.048725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772963.048734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772963.176793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772963.176805:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772963.176811:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772963.176812:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772963.240742:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772963.240746:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772963.240746:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772963.240750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772963.240806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772963.240812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772963.368725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772963.432724:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772963.496742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772963.560728:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772963.560741:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772963.560748:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772963.560756:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772963.624737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772963.624799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772963.624863:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772963.752748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772963.752759:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772963.816737:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772963.880795:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772963.944724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772963.944741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772963.944741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772964.008731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772964.072788:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772964.072788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772964.072794:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772964.200732:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772964.200732:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772964.200732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772964.200788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772964.264733:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772964.264779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772964.264823:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772964.264823:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772964.264826:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772964.264846:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772964.392724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772964.456749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772964.520738:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772964.584741:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772964.584743:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772964.584771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772964.584776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772964.648720:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772964.648764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772964.648944:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772964.776725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772964.776725:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772964.840776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772964.904726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772964.968766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772964.968766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772964.968778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772965.032723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772965.096764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772965.096772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772965.096775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772965.224762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772965.224770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772965.224789:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772965.224793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772965.288750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772965.288776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772965.288781:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772965.288805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772965.288815:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772965.288840:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772965.416705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772965.480725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772965.544783:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772965.608770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772965.608774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772965.608797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772965.608802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772965.672788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790772965.672792:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772965.672867:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772965.800755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772965.800755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772965.864749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772965.928788:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772965.992760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772965.992760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772965.992770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772966.056771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772966.120778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772966.120778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772966.120781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772966.248757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772966.248804:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772966.248805:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772966.248805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772966.312760:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772966.312767:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772966.312767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772966.312767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772966.312802:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772966.312807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772966.440737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772966.504705:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772966.568760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772966.632740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772966.632754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772966.632782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772966.632784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772966.696792:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772966.696808:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772966.696823:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772966.824748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772966.824748:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772966.888758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772966.952781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772967.016753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772967.016753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772967.016753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772967.080717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772967.144755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772967.144759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772967.144764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772967.272760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772967.272761:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772967.272761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772967.272761:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772967.336732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772967.336747:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772967.336747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772967.336779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772967.336795:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772967.336818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772967.464728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772967.528750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772967.592740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772967.592776:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772967.592782:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772967.592786:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772967.592794:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1963 00000100:00080000:3.0:1790772967.592803:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772967.592813:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772967.592847:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534570240/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772967.592871:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772967.592876:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772967.592878:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772967.592882:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1963 00000100:00080000:3.0:1790772967.592886:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772967.592893:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772967.592903:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772967.592917:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534570368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772967.592928:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772967.592932:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772967.592935:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772967.592940:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1963 00000100:00080000:3.0:1790772967.592944:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772967.592948:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772967.592953:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772967.592963:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534570496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772967.592972:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772967.592976:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772967.592977:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772967.592981:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1963 00000100:00080000:3.0:1790772967.592985:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772967.592989:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772967.592994:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790772967.593003:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534570624/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772967.593012:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772967.593016:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772967.593018:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772967.593022:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1963 00000100:00080000:3.0:1790772967.593026:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772967.593030:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772967.593035:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772967.593046:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534570752/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772967.593056:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772967.593060:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772967.593062:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772967.593066:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1963 00000100:00080000:3.0:1790772967.593070:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772967.593076:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772967.593081:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772967.593092:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534570880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772967.593098:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534570240:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772967.593102:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772967.593105:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772967.593107:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772967.593111:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1963 00000100:00080000:3.0:1790772967.593115:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772967.593119:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772967.593123:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772967.593132:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534571008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772967.593140:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772967.593143:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772967.593145:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772967.593149:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1963 00000100:00080000:3.0:1790772967.593153:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772967.593157:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790772967.593161:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790772967.593161:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790772967.593170:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534571136/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772967.593191:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772967.593194:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772967.593196:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772967.593200:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1963 00000100:00100000:0.0:1790772967.593203:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772967/real 1790772967] req@ffff9c4ef73c2d80 x1877759534570240/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772967.593204:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772967.593209:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772967.593213:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772967.593221:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534570240/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772967.593223:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534571264/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772967.593233:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772967.593237:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772967.593241:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534570240:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790772967.593249:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534570368:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772967.593274:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772967.593287:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772967/real 1790772967] req@ffff9c4ef73c1500 x1877759534570368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772967.593297:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534570368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772967.593310:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772967.593313:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772967.593316:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534570368:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772967.593321:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534570496:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772967.593336:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772967.593343:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772967/real 1790772967] req@ffff9c4ef73c0a80 x1877759534570496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772967.593351:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534570496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772967.593359:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772967.593362:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772967.593366:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534570496:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772967.593373:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534570624:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772967.593393:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772967.593406:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772967/real 1790772967] req@ffff9c4ef73c3800 x1877759534570624/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772967.593419:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534570624/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772967.593431:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772967.593435:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772967.593439:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534570624:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772967.593446:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534570752:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772967.593464:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772967.593473:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772967/real 1790772967] req@ffff9c4ef73c2a00 x1877759534570752/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772967.593481:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534570752/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772967.593488:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772967.593490:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772967.593493:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534570752:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772967.593515:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534570880:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772967.593527:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772967.593534:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772967/real 1790772967] req@ffff9c4ef73c0700 x1877759534570880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772967.593542:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534570880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772967.593553:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772967.593557:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772967.593560:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534570880:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772967.593567:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534571008:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772967.593585:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772967.593597:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772967/real 1790772967] req@ffff9c4ef73c2300 x1877759534571008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772967.593611:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534571008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772967.593627:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772967.593629:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772967.593632:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534571008:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772967.593637:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534571136:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772967.593670:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772967.593679:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772967/real 1790772967] req@ffff9c4ef73c0000 x1877759534571136/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772967.593687:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534571136/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772967.593712:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772967.593715:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772967.593717:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534571136:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772967.593722:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534571264:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772967.593737:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772967.593745:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772967/real 1790772967] req@ffff9c4ef73c1c00 x1877759534571264/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772967.593753:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534571264/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772967.593760:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772967.593762:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772967.593765:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534571264:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772967.656751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772967.656751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772967.656751:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772967.656751:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772967.720756:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772967.720791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772967.720807:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772967.848744:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772967.848745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772967.912740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772967.976740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772968.040739:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772968.040740:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772968.040740:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772968.104726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772968.168762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772968.168762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772968.168765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772968.296726:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772968.296743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772968.296747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772968.296762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772968.360776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772968.360782:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772968.360786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772968.360808:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772968.360822:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772968.360842:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772968.488750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772968.552749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772968.616745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772968.680734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772968.680787:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772968.680788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772968.680792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772968.744770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772968.744805:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772968.744855:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772968.872787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772968.872787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772968.936746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772969.000743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772969.064757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772969.064757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772969.064758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772969.128746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772969.192735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772969.192738:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772969.192742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772969.320778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772969.320778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772969.320779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772969.320803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772969.384749:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772969.384750:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772969.384762:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772969.384769:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772969.384814:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772969.384814:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772969.512735:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772969.576765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772969.640734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772969.704746:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772969.704753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772969.704753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772969.704782:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772969.768743:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772969.768779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772969.768816:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772969.896728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772969.896745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772969.960745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772970.024810:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772970.088730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772970.088730:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772970.088733:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772970.152765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772970.216737:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772970.216737:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772970.216738:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772970.344755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772970.344755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772970.344777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772970.344778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772970.408722:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772970.408722:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772970.408744:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772970.408760:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772970.408781:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772970.408788:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772970.536722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772970.600751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772970.664722:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772970.728730:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772970.728739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772970.728742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772970.728799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772970.792724:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772970.792776:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772970.792838:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772970.920733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772970.920733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772970.984747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772971.048752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772971.112752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772971.112765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772971.112766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772971.176715:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772971.240716:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772971.240716:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772971.240734:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772971.368799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772971.368799:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772971.368799:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772971.368800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772971.432713:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772971.432713:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772971.432746:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772971.432764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772971.432781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772971.432804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772971.560712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772971.624762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772971.688730:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772971.752759:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772971.752767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772971.752767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772971.752771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772971.816734:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772971.816786:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772971.816843:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772971.944727:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772971.944727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772972.008748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772972.072722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772972.136732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772972.136732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772972.136783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772972.200717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772972.264744:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772972.264753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772972.264753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772972.392722:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772972.392722:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772972.392726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772972.392826:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772972.456734:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772972.456745:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772972.456746:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772972.456781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772972.456807:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772972.456832:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772972.584724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772972.648713:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790772972.712740:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772972.712747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790772972.712750:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790772972.712757:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772972.712766:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1968 00000100:00080000:3.0:1790772972.712775:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772972.712785:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790772972.712820:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534571392/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772972.712844:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772972.712848:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772972.712849:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772972.712853:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1968 00000100:00080000:3.0:1790772972.712856:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772972.712859:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772972.712868:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772972.712878:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534571520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772972.712884:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534571392:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772972.712889:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772972.712893:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772972.712894:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772972.712898:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1968 00000100:00080000:3.0:1790772972.712900:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772972.712903:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772972.712907:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790772972.712917:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534571648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772972.712924:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772972.712927:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772972.712928:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772972.712930:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1968 00000100:00080000:3.0:1790772972.712933:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772972.712937:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772972.712940:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:0.0:1790772972.712946:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772972.712950:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534571776/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772972.712958:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772972.712961:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772972.712963:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772972.712967:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1968 00000100:00080000:3.0:1790772972.712969:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772972.712972:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772972.712976:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790772972.712985:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534571904/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772972.712990:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772972/real 1790772972] req@ffff9c4ef73c0e00 x1877759534571392/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772972.712995:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772972.713003:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790772972.713006:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772972.713012:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1968 00000100:00080000:3.0:1790772972.713015:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772972.713017:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534571392/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772972.713022:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772972.713030:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790772972.713034:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790772972.713039:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534572032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772972.713041:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772972.713049:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534571392:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790772972.713054:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772972.713057:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790772972.713059:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772972.713063:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1968 00000100:00080000:3.0:1790772972.713065:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790772972.713068:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772972.713072:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772972.713075:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534571520:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790772972.713083:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534572160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772972.713092:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790772972.713097:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790772972.713098:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790772972.713102:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790772972.713107:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1968 00000100:00080000:3.0:1790772972.713110:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772972.713113:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772972/real 1790772972] req@ffff9c4ef73c1c00 x1877759534571520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790772972.713117:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790772972.713123:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772972.713132:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534571520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790772972.713137:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534572288/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790772972.713151:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790772972.713154:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790772972.713156:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790772972.713157:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790772972.713161:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1968 00000100:00080000:0.0:1790772972.713165:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790772972.713168:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772972.713170:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534571520:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790772972.713179:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772972.713180:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534571648:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790772972.713189:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:0.0:1790772972.713200:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790772972.713208:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534572416/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772972.713211:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772972/real 1790772972] req@ffff9c4ef73c0000 x1877759534571648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772972.713242:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534571648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772972.713250:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772972.713253:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772972.713256:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534571648:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772972.713261:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534571776:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772972.713274:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772972.713280:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772972/real 1790772972] req@ffff9c4ef73c2300 x1877759534571776/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772972.713289:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534571776/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772972.713296:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772972.713298:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772972.713300:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534571776:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772972.713304:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534571904:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772972.713314:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772972.713320:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772972/real 1790772972] req@ffff9c4ef73c0700 x1877759534571904/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772972.713328:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534571904/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772972.713339:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772972.713342:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772972.713344:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534571904:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772972.713347:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534572032:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772972.713357:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772972.713363:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772972/real 1790772972] req@ffff9c4ef73c2a00 x1877759534572032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772972.713370:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534572032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772972.713382:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772972.713384:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772972.713386:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534572032:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772972.713399:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534572160:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772972.713410:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772972.713416:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772972/real 1790772972] req@ffff9c4ef73c3800 x1877759534572160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772972.713424:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534572160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772972.713429:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772972.713432:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772972.713434:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534572160:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772972.713438:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534572288:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772972.713448:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772972.713463:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772972/real 1790772972] req@ffff9c4ef73c0a80 x1877759534572288/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772972.713471:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534572288/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772972.713479:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772972.713482:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772972.713484:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534572288:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772972.713488:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534572416:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772972.713500:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772972.713506:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772972/real 1790772972] req@ffff9c4ef73c1500 x1877759534572416/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772972.713514:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534572416/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790772997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772972.713521:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772972.713523:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772972.713525:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534572416:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772972.776745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772972.776747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772972.776750:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772972.776808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772972.840732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772972.840739:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772972.840900:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772972.968724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772972.968724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772973.032797:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772973.096724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772973.160729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772973.160729:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772973.160772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772973.224699:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772973.288763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772973.288809:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772973.288826:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772973.416734:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772973.416788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772973.416803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772973.416804:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772973.480744:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772973.480747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772973.480760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772973.480785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772973.480805:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772973.480823:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772973.608725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772973.672773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772973.736719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772973.800733:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772973.800733:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772973.800734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772973.800734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772973.864733:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772973.864756:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772973.864786:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772973.992723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772973.992734:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772974.056755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772974.120726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772974.184711:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772974.184713:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772974.184723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772974.248711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772974.312732:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772974.312732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772974.312732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772974.440708:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772974.440714:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772974.440722:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772974.440732:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772974.504724:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772974.504731:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772974.504731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772974.504752:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772974.504771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772974.504771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772974.632714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772974.696740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772974.760724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772974.824750:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772974.824758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772974.824764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772974.824765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772974.888751:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772974.888794:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772974.888915:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772975.016707:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772975.016732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772975.080733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772975.144733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772975.208732:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772975.208754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772975.208768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772975.272726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772975.336747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772975.336750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772975.336752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772975.464728:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772975.464740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772975.464740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772975.464741:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772975.528731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772975.528734:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772975.528753:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772975.528775:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772975.528785:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772975.528795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772975.656705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772975.720730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772975.784698:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772975.848703:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772975.848724:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772975.848731:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772975.848732:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772975.912748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772975.912748:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772975.912784:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772976.040730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772976.040732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772976.104723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772976.168722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772976.232707:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772976.232718:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772976.232721:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772976.296701:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772976.360740:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772976.360740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772976.360742:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772976.488715:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772976.488721:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772976.488726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772976.488729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772976.552704:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772976.552726:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772976.552733:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772976.552738:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772976.552756:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772976.552761:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772976.680709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772976.744723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772976.808725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772976.872739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772976.872739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772976.872740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772976.872740:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772976.936707:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772976.936734:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772976.936789:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772977.064731:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772977.064731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772977.128766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772977.192790:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772977.256722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772977.256722:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772977.256748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772977.320700:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772977.384718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772977.384721:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772977.384722:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772977.512717:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772977.512717:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772977.512733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772977.512827:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772977.576713:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772977.576728:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772977.576731:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772977.576733:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772977.576753:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772977.576757:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772977.704717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772977.768753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772977.832713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772977.832744:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772977.832749:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772977.832753:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772977.832762:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1973 00000100:00080000:1.0:1790772977.832772:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772977.832783:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772977.832817:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534572544/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772977.832838:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772977.832841:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772977.832843:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772977.832846:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1973 00000100:00080000:1.0:1790772977.832848:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772977.832852:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772977.832859:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772977.832870:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534572672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772977.832880:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534572544:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772977.832885:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772977.832890:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772977.832892:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772977.832898:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1973 00000100:00080000:1.0:1790772977.832903:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772977.832908:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772977.832914:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772977.832930:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534572800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772977.832941:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772977.832945:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772977.832949:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790772977.832951:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772977.832958:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1973 00000100:00080000:1.0:1790772977.832964:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772977.832972:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772977.832978:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772977.832992:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534572928/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772977.833000:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772977.833003:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772977/real 1790772977] req@ffff9c4efac4df80 x1877759534572544/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772977.833003:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772977.833007:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772977.833013:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1973 00000100:00080000:1.0:1790772977.833020:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772977.833029:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772977.833035:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772977.833036:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534572544/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772977.833053:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790772977.833054:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534573056/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772977.833063:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772977.833073:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534572544:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772977.833074:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772977.833084:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772977.833087:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772977.833092:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1973 00000100:00080000:1.0:1790772977.833097:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772977.833104:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772977.833107:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534572672:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790772977.833112:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772977.833129:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534573184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790772977.833143:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772977.833143:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772977.833149:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772977.833152:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772977.833157:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1973 00000100:00080000:1.0:1790772977.833162:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772977.833169:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772977.833175:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772977.833176:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772977/real 1790772977] req@ffff9c4efac4c700 x1877759534572672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772977.833192:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534573312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772977.833198:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534572672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772977.833203:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772977.833212:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772977.833214:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772977.833220:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1973 00000100:00080000:1.0:1790772977.833228:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772977.833236:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790772977.833243:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790772977.833244:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790772977.833253:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772977.833260:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534572672:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772977.833263:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534573440/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772977.833273:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534572800:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772977.833279:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772977.833285:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772977.833287:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772977.833294:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1973 00000100:00080000:1.0:1790772977.833301:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790772977.833307:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772977.833309:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772977.833317:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772977.833326:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772977/real 1790772977] req@ffff9c4efac4ed80 x1877759534572800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772977.833333:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534573568/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772977.833375:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534572800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772977.833386:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772977.833390:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772977.833394:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534572800:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772977.833401:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534572928:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772977.833424:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772977.833435:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772977/real 1790772977] req@ffff9c4efac4dc00 x1877759534572928/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772977.833447:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534572928/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772977.833457:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772977.833460:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772977.833464:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534572928:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772977.833470:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534573056:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772977.833489:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772977.833520:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772977/real 1790772977] req@ffff9c4efac4f800 x1877759534573056/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772977.833533:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534573056/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772977.833549:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772977.833553:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772977.833557:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534573056:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772977.833576:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534573184:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772977.833598:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772977.833610:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772977/real 1790772977] req@ffff9c4efac4f480 x1877759534573184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772977.833621:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534573184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772977.833637:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772977.833641:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772977.833645:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534573184:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772977.833672:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534573312:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772977.833695:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772977.833707:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772977/real 1790772977] req@ffff9c4efac4e680 x1877759534573312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772977.833718:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534573312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772977.833727:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772977.833730:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772977.833735:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534573312:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772977.833741:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534573440:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772977.833760:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772977.833770:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772977/real 1790772977] req@ffff9c4efac4ea00 x1877759534573440/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772977.833782:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534573440/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772977.833794:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772977.833797:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772977.833801:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534573440:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772977.833807:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534573568:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772977.833829:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772977.833842:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772977/real 1790772977] req@ffff9c4efac4e300 x1877759534573568/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772977.833855:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534573568/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772977.833867:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772977.833871:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772977.833875:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534573568:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772977.896734:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772977.896737:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772977.896738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772977.896739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772977.960729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772977.960736:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772977.960797:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772978.088721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772978.088725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772978.152721:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772978.216708:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772978.280740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772978.280740:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772978.280740:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772978.344722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772978.408723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772978.408729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772978.408731:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772978.536739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772978.536743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772978.536744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772978.536747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772978.600732:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772978.600733:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772978.600734:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772978.600758:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772978.600784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772978.600788:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772978.728714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772978.792722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772978.856705:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772978.920739:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772978.920739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772978.920740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772978.920744:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772978.984709:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772978.984714:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772978.984795:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772979.112718:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772979.112720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772979.176769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772979.240730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772979.304722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772979.304722:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772979.304733:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772979.368713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772979.432753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772979.432754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772979.432754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772979.560752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772979.560752:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772979.560753:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772979.560762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772979.624724:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772979.624725:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772979.624767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772979.624767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772979.624786:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772979.624786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772979.752712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772979.816738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772979.880737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772979.944766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772979.944807:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772979.944815:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772979.944816:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772980.008754:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772980.008759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772980.008829:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772980.136717:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772980.136723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772980.200722:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772980.264751:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772980.328754:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772980.328754:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772980.328754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772980.392721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772980.456739:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772980.456739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772980.456739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772980.584752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772980.584758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772980.584771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772980.584780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772980.648697:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772980.648772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772980.648772:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772980.648772:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772980.648772:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772980.648804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772980.776716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772980.840714:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772980.904706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772980.968728:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772980.968735:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772980.968739:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772980.968750:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772981.032708:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772981.032728:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772981.032835:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772981.160710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772981.160711:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772981.224766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772981.288736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772981.352718:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772981.352726:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772981.352764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772981.416706:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772981.480718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772981.480723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772981.480747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772981.608714:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772981.608720:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772981.608721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772981.608722:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772981.672733:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772981.672733:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772981.672736:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772981.672747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772981.672765:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772981.672776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772981.800707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772981.864715:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772981.928704:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772981.992739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772981.992746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772981.992747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772981.992772:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772982.056743:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772982.056786:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772982.056912:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772982.184695:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772982.184710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772982.248734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772982.312730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772982.376723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772982.376723:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772982.376724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772982.440704:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772982.504704:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772982.504722:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772982.504725:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772982.632713:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772982.632716:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772982.632721:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772982.632740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790772982.696728:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772982.696732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772982.696735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772982.696747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772982.696760:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772982.696772:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772982.824719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772982.888722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772982.952713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772982.952732:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772982.952739:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772982.952744:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772982.952754:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1978 00000100:00080000:1.0:1790772982.952764:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772982.952776:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772982.952814:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534573696/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772982.952844:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772982.952848:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772982.952850:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772982.952854:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1978 00000100:00080000:1.0:1790772982.952859:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772982.952865:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772982.952875:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772982.952891:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534573696:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772982.952894:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534573824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772982.952910:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772982.952913:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772982.952916:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772982.952920:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1978 00000100:00080000:1.0:1790772982.952923:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772982.952929:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772982.952935:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772982.952949:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534573952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790772982.952958:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772982.952964:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772982.952969:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772982.952971:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772982.952978:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1978 00000100:00080000:1.0:1790772982.952983:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772982.952988:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772982.952994:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790772982.953007:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772982/real 1790772982] req@ffff9c4efac4fb80 x1877759534573696/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772982.953010:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534574080/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772982.953035:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772982.953039:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1790772982.953043:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534573696/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772982.953044:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772982.953055:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1978 00000100:00080000:0.0:1790772982.953058:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772982.953061:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790772982.953066:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772982.953073:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534573696:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772982.953073:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772982.953085:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772982.953102:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534573824:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772982.953103:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534574208/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772982.953119:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772982.953123:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:0.0:1790772982.953128:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772982.953128:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772982.953137:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1978 00000100:00080000:1.0:1790772982.953142:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772982.953143:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772982/real 1790772982] req@ffff9c4efac4e300 x1877759534573824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772982.953151:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772982.953159:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534573824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772982.953160:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772982.953180:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534574336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772982.953182:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772982.953190:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772982.953197:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534573824:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772982.953202:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772982.953206:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534573952:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772982.953209:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772982.953214:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772982.953220:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1978 00000800:00000100:0.0:1790772982.953225:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772982.953226:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772982.953237:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772982.953238:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772982/real 1790772982] req@ffff9c4efac4ea00 x1877759534573952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772982.953245:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772982.953260:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534573952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772982.953265:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534574464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772982.953273:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772982.953281:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772982.953284:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772982.953285:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534573952:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772982.953294:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1790772982.953295:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534574080:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790772982.953299:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772982.953306:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1978 00000100:00080000:1.0:1790772982.953312:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790772982.953315:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772982.953322:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790772982.953327:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772982/real 1790772982] req@ffff9c4efac4e680 x1877759534574080/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790772982.953331:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790772982.953339:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534574080/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772982.953348:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790772982.953350:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534574592/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790772982.953354:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772982.953360:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534574080:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790772982.953373:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772982.953381:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772982.953383:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772982.953389:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1978 00000100:00080000:1.0:1790772982.953408:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772982.953415:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772982.953420:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772982.953432:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534574720/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772982.953525:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534574208:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772982.953544:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772982.953553:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772982/real 1790772982] req@ffff9c4efac4f480 x1877759534574208/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772982.953568:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534574208/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772982.953580:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772982.953583:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772982.953585:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534574208:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772982.953589:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534574336:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772982.953602:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772982.953609:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772982/real 1790772982] req@ffff9c4efac4f800 x1877759534574336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772982.953617:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534574336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772982.953628:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772982.953630:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772982.953633:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534574336:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772982.953637:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534574464:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790772982.953647:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772982.953675:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772982/real 1790772982] req@ffff9c4efac4dc00 x1877759534574464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772982.953683:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534574464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772982.953689:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772982.953692:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772982.953694:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534574464:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790772982.953699:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534574592:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772982.953711:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772982.953718:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772982/real 1790772982] req@ffff9c4efac4ed80 x1877759534574592/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772982.953726:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534574592/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772982.953734:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772982.953736:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772982.953738:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534574592:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772982.953748:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534574720:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790772982.953760:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790772982.953766:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772982/real 1790772982] req@ffff9c4efac4c700 x1877759534574720/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790772982.953774:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534574720/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790772982.953780:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790772982.953782:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790772982.953784:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534574720:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772983.016729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772983.016731:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772983.016734:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772983.016737:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772983.080705:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772983.080781:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772983.080826:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772983.208711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772983.208711:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772983.272756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772983.336736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772983.400716:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772983.400732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772983.400733:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772983.464703:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772983.528748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772983.528748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772983.528748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772983.656761:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772983.656762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772983.656762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772983.656762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772983.720735:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790772983.720735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772983.720735:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772983.720766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772983.720771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772983.720785:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772983.848712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772983.912718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772983.976709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772984.040712:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772984.040714:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772984.040732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772984.040732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772984.104724:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772984.104734:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772984.104808:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772984.232721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772984.232728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772984.296770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772984.360773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772984.424751:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772984.424751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772984.424751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772984.488694:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772984.552723:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772984.552723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772984.552726:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772984.680725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772984.680729:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772984.680736:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772984.680736:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772984.744715:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772984.744736:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772984.744741:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772984.744748:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772984.744759:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772984.744775:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772984.872706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772984.936733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772985.000716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772985.064739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772985.064739:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772985.064743:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772985.064765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772985.128717:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772985.128746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772985.128891:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772985.256718:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772985.256729:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772985.320769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772985.384737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772985.448743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772985.448744:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772985.448756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772985.512717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772985.576795:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772985.576795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772985.576799:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772985.704721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772985.704766:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772985.704797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772985.704803:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772985.768709:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772985.768722:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772985.768732:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772985.768744:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772985.768755:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772985.768760:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772985.896705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772985.960708:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772986.024711:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772986.088723:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772986.088728:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772986.088728:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772986.088735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772986.152725:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772986.152768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772986.152798:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772986.280723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772986.280811:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772986.344723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772986.408725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772986.472719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772986.472719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772986.472721:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772986.536711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772986.600704:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772986.600730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772986.600730:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772986.728764:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772986.728765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772986.728775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772986.728792:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772986.792722:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772986.792731:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772986.792758:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772986.792772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772986.792776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772986.792777:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772986.920747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772986.984719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772987.048721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772987.112741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772987.112742:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772987.112742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772987.112742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772987.176733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772987.176747:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772987.176804:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772987.304711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772987.304713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772987.368744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772987.432723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772987.496728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772987.496729:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772987.496729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772987.560719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772987.624753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772987.624753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772987.624759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772987.752724:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772987.752730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772987.752739:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772987.752764:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772987.816737:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772987.816737:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772987.816748:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772987.816748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772987.816772:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772987.816772:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772987.944715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772988.008715:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790772988.072734:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772988.072736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772988.072743:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772988.072747:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772988.072758:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1983 00000100:00080000:1.0:1790772988.072768:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772988.072781:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772988.072817:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534574848/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772988.072842:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772988.072845:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772988.072847:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772988.072850:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1983 00000100:00080000:1.0:1790772988.072853:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772988.072858:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772988.072866:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790772988.072874:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534574848:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772988.072879:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534574976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772988.072888:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772988.072890:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772988.072892:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772988.072895:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1983 00000100:00080000:1.0:1790772988.072897:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772988.072900:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772988.072904:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772988.072913:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534575104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772988.072919:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772988.072922:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772988.072924:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772988.072926:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1983 00000100:00080000:1.0:1790772988.072929:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772988.072932:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772988.072936:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772988.072945:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534575232/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772988.072956:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772988.072960:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772988.072962:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772988.072967:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1983 00000100:00100000:0.0:1790772988.072970:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534574976:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772988.072973:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772988.072981:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772988.072988:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790772988.073001:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534575104:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772988.073003:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534575360/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772988.073017:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790772988.073021:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534575232:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790772988.073024:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772988.073028:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772988.073035:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1983 00000100:00080000:1.0:1790772988.073040:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790772988.073044:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534575360:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790772988.073049:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772988.073056:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772988.073071:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534575488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772988.073088:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772988.073092:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772988.073094:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772988.073099:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1983 00000100:00100000:0.0:1790772988.073103:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534575488:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772988.073105:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772988.073115:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772988.073120:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772988.073137:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534575616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772988.073151:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772988.073155:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772988.073158:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790772988.073162:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534575616:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772988.073167:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1983 00000100:00080000:1.0:1790772988.073173:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772988.073187:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772988.073192:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790772988.073202:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534575744/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772988.073215:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772988.073218:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772988.073219:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772988.073222:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1983 00000100:00080000:1.0:1790772988.073225:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772988.073228:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772988.073232:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790772988.073241:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534575744:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772988.073242:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534575872/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790772988.073282:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534575872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790772988.136721:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772988.136721:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772988.136727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772988.136729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772988.200727:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772988.200730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772988.200794:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772988.328719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772988.328727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772988.392723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772988.456729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772988.520742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772988.520742:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772988.520742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772988.584704:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772988.648714:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772988.648719:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772988.648719:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772988.776730:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772988.776731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772988.776731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772988.776731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772988.840705:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772988.840724:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772988.840727:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772988.840732:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772988.840758:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772988.840758:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772988.968710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772989.032731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772989.096779:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772989.160726:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772989.160726:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772989.160728:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772989.160729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772989.224709:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772989.224724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772989.224789:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772989.352724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772989.352736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772989.416724:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772989.480718:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772989.544719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772989.544727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772989.544728:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772989.608707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772989.672730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772989.672730:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772989.672730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772989.800697:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772989.800717:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772989.800720:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772989.800722:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772989.864755:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772989.864777:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772989.864808:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772989.864817:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772989.864826:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772989.864843:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772989.992764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772990.056747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772990.120776:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772990.184744:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772990.184744:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772990.184745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772990.184768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772990.248755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790772990.248800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772990.248830:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772990.376723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772990.376731:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772990.440722:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772990.504712:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772990.568717:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772990.568720:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772990.568720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772990.632753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772990.696716:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772990.696725:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772990.696736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772990.824786:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772990.824786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772990.824797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772990.824827:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772990.888738:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772990.888789:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772990.888791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772990.888811:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772990.888838:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772990.888858:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772991.016705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772991.080724:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772991.144874:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:0.0:1790772991.144994:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:0.0:1790772991.145040:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1790772991.145058:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772991.145111:0:2383:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:0.0:1790772991.145127:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772991.145138:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772991.145150:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772991.145166:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772991.145180:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790772991.145185:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772988/real 1790772991] req@ffff9c4efac4e300 x1877759534575872/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1790772991.145198:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790772991.145216:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790772991.145254:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534575872/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772991.145276:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772991.145284:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772991.145291:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534575872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772991.145302:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772988/real 1790772991] req@ffff9c4efac4ea00 x1877759534575744/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772991.145321:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534575744/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772991.145335:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772991.145340:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772991.145345:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534575744:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772991.145354:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772988/real 1790772991] req@ffff9c4efac4e680 x1877759534575616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772991.145370:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534575616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772991.145381:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772991.145385:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772991.145390:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534575616:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772991.145399:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772988/real 1790772991] req@ffff9c4efac4f480 x1877759534575488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772991.145415:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534575488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772991.145438:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772991.145443:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772991.145449:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534575488:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772991.145459:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772988/real 1790772991] req@ffff9c4efac4f800 x1877759534575360/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772991.145476:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534575360/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772991.145489:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772991.145494:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772991.145499:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534575360:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772991.145509:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772988/real 1790772991] req@ffff9c4efac4c700 x1877759534574976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772991.145526:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534574976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772991.145546:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772991.145553:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772991.145558:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534574976:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772991.145567:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772988/real 1790772991] req@ffff9c4efac4ed80 x1877759534575104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772991.145584:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534575104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772991.145595:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772991.145600:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772991.145604:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534575104:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772991.145613:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772988/real 1790772991] req@ffff9c4efac4dc00 x1877759534575232/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772991.145627:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534575232/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772991.145638:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772991.145643:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772991.145647:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534575232:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772991.145723:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772988/real 1790772991] req@ffff9c4efac4df80 x1877759534574848/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772991.145743:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534574848/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772991.145755:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772991.145761:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772991.145767:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534574848:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790772991.208728:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772991.208732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772991.208736:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772991.208762:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772991.272747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772991.272810:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772991.272849:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772991.400723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772991.400723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772991.464725:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772991.528797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772991.592750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772991.592750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772991.592774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772991.656718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772991.720759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772991.720760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772991.720774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772991.848715:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772991.848725:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772991.848726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772991.848766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772991.912711:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772991.912741:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772991.912741:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772991.912752:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772991.912773:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772991.912780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772992.040715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772992.104705:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772992.168765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772992.232738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772992.232738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772992.232739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772992.232739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772992.296734:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772992.296748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772992.296785:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772992.424721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772992.424722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772992.488722:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772992.552723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772992.616734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772992.616734:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772992.616734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772992.680756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772992.744712:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772992.744715:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772992.744729:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772992.872728:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772992.872728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772992.872735:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772992.872736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772992.936704:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772992.936717:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772992.936735:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772992.936736:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772992.936749:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772992.936753:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772993.064707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772993.128714:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790772993.192719:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772993.192728:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772993.192732:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772993.192740:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1989 00000100:00080000:1.0:1790772993.192749:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772993.192759:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772993.192787:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772993.192791:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534576000/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772993.192813:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772993.192817:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772993.192818:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772993.192822:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1989 00000100:00080000:1.0:1790772993.192825:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772993.192829:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772993.192837:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772993.192845:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534576000:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772993.192848:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534576128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772993.192859:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772993.192862:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772993.192863:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772993.192867:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1989 00000100:00080000:1.0:1790772993.192870:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772993.192873:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772993.192877:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772993.192885:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534576256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772993.192892:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772993.192895:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772993.192897:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772993.192899:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1989 00000100:00080000:1.0:1790772993.192902:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772993.192906:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772993.192907:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772993.192912:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772993.192921:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534576384/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772993.192929:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772993.192932:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772993.192933:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772993.192936:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1989 00000100:00080000:1.0:1790772993.192938:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772993.192942:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772993.192945:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790772993.192957:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534576512/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772993.192959:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772993/real 1790772993] req@ffff9c4efac4fb80 x1877759534576000/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772993.192972:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772993.192980:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772993.192983:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772993.192987:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1989 00000100:00080000:1.0:1790772993.192990:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772993.192994:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772993.192999:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772993.192999:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534576000/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772993.193011:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534576640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790772993.193013:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772993.193022:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772993.193024:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772993.193029:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772993.193032:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772993.193033:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534576000:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772993.193039:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1989 00000100:00080000:1.0:1790772993.193045:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772993.193052:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772993.193057:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772993.193067:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534576128:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772993.193072:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534576768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772993.193084:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772993.193088:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772993.193091:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772993.193096:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1989 00000100:00080000:1.0:1790772993.193102:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790772993.193103:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772993.193110:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772993.193116:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772993.193125:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772993/real 1790772993] req@ffff9c4efac4df80 x1877759534576128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772993.193132:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534576896/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772993.193151:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790772993.193152:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534576128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772993.193165:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772993.193169:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772993.193175:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1989 00000100:00080000:2.0:1790772993.193178:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772993.193181:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772993.193186:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772993.193191:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772993.193212:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534576128:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790772993.193221:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790772993.193223:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534576256:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790772993.193240:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534577024/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790772993.193258:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772993.193277:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772993/real 1790772993] req@ffff9c4efac4dc00 x1877759534576256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772993.193291:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534576256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772993.193301:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772993.193305:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772993.193309:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534576256:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772993.193316:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534576384:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772993.193337:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772993.193349:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772993/real 1790772993] req@ffff9c4efac4ed80 x1877759534576384/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772993.193362:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534576384/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772993.193373:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772993.193376:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772993.193380:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534576384:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772993.193387:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534576512:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772993.193407:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772993.193420:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772993/real 1790772993] req@ffff9c4efac4c700 x1877759534576512/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772993.193433:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534576512/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772993.193442:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772993.193445:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772993.193449:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534576512:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772993.193456:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534576640:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772993.193476:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772993.193489:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772993/real 1790772993] req@ffff9c4efac4f800 x1877759534576640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772993.193502:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534576640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772993.193519:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772993.193523:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772993.193527:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534576640:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772993.193551:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534576768:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772993.193573:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772993.193585:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772993/real 1790772993] req@ffff9c4efac4f480 x1877759534576768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772993.193597:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534576768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772993.193609:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772993.193613:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772993.193618:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534576768:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772993.193624:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534576896:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772993.193682:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772993.193702:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772993/real 1790772993] req@ffff9c4efac4e680 x1877759534576896/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772993.193716:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534576896/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772993.193729:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772993.193734:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772993.193738:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534576896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772993.193746:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534577024:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772993.193775:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772993.193788:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772993/real 1790772993] req@ffff9c4efac4ea00 x1877759534577024/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772993.193800:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534577024/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772993.193809:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772993.193813:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772993.193818:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534577024:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772993.256719:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772993.256728:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790772993.256731:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772993.256731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772993.320721:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772993.320726:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790772993.320777:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772993.448729:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772993.448729:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772993.512717:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772993.576728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772993.640714:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772993.640720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772993.640722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772993.704723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790772993.768735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772993.768735:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772993.768759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772993.896729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772993.896729:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772993.896730:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790772993.896733:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772993.960721:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772993.960726:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772993.960726:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772993.960762:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772993.960780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772993.960782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772994.088705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772994.152730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772994.216798:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772994.280744:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772994.280744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772994.280744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772994.280745:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772994.344712:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772994.344799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790772994.344809:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772994.472710:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772994.472739:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772994.536756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772994.600717:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772994.664720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772994.664732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772994.664771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772994.728707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772994.792719:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772994.792720:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772994.792754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772994.920739:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772994.920739:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772994.920739:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772994.920757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772994.984718:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772994.984734:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772994.984736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772994.984738:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772994.984759:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772994.984765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772995.112714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772995.176736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772995.240708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772995.304720:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772995.304720:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772995.304726:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772995.304727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772995.368735:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772995.368740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772995.368772:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772995.496749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772995.496749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772995.560738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772995.624721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772995.688713:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772995.688726:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772995.688732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772995.752717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790772995.816731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772995.816731:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772995.816753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772995.944723:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772995.944726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772995.944730:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772995.944731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772996.008727:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772996.008727:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772996.008729:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772996.008740:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772996.008770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790772996.008782:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772996.136703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772996.200713:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772996.264721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772996.328792:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772996.328810:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772996.328815:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772996.328815:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772996.392771:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772996.392782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772996.392866:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772996.520735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772996.520735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772996.584727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772996.648755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772996.712747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790772996.712765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772996.712771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772996.776702:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772996.840744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772996.840748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772996.840748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772996.968711:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772996.968725:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790772996.968730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772996.968734:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772997.032740:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772997.032741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790772997.032741:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790772997.032799:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772997.032810:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772997.032813:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772997.160726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772997.224721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772997.288707:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772997.352720:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790772997.352728:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772997.352730:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772997.352730:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772997.416725:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790772997.416770:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790772997.416780:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772997.544711:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772997.544736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772997.608720:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772997.672725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772997.736714:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772997.736737:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772997.736744:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772997.800712:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772997.864716:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772997.864731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772997.864742:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772997.992714:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790772997.992721:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790772997.992722:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772997.992722:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772998.056730:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772998.056740:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772998.056743:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772998.056814:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790772998.056819:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772998.056837:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790772998.184712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772998.248726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772998.312735:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790772998.312750:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772998.312762:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790772998.312767:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772998.312779:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1994 00000100:00080000:1.0:1790772998.312791:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772998.312803:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790772998.312842:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534577152/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772998.312870:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772998.312875:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772998.312877:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772998.312882:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1994 00000100:00080000:1.0:1790772998.312888:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772998.312896:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772998.312906:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772998.312921:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534577280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772998.312928:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534577152:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790772998.312950:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772998.312955:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772998.312958:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772998.312964:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1994 00000100:00080000:1.0:1790772998.312980:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772998.312987:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772998.312993:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790772998.313010:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534577408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790772998.313022:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772998.313026:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790772998.313029:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790772998.313036:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790772998.313042:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1994 00000100:00080000:1.0:1790772998.313050:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772998.313059:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772998.313066:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790772998.313083:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534577536/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772998.313108:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772998/real 1790772998] req@ffff9c4efac4e300 x1877759534577152/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772998.313111:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772998.313128:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772998.313134:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772998.313141:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1994 00000100:00100000:2.0:1790772998.313144:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534577152/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772998.313152:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772998.313159:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772998.313165:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790772998.313172:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790772998.313178:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790772998.313182:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534577152:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790772998.313199:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534577664/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772998.313220:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534577280:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772998.313221:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772998.313232:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790772998.313235:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790772998.313243:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1994 00000100:00080000:1.0:1790772998.313249:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772998.313259:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772998.313263:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772998.313268:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790772998.313288:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772998/real 1790772998] req@ffff9c4efac4ea00 x1877759534577280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772998.313289:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534577792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772998.313339:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534577280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772998.313341:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772998.313366:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790772998.313370:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772998.313377:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772998.313379:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1994 00000100:00080000:2.0:1790772998.313383:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772998.313391:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790772998.313392:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534577280:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772998.313410:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534577408:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772998.313411:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790772998.313429:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790772998.313442:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790772998.313449:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534577920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772998.313454:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772998/real 1790772998] req@ffff9c4efac4e680 x1877759534577408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772998.313504:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534577408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772998.313511:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790772998.313515:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772998.313518:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790772998.313521:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772998.313523:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790772998.313527:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534577408:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790772998.313536:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1994 00000100:00100000:2.0:1790772998.313537:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534577536:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790772998.313550:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790772998.313559:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790772998.313566:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790772998.313569:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790772998.313578:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772998/real 1790772998] req@ffff9c4efac4f480 x1877759534577536/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790772998.313588:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534578048/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772998.313595:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534577536/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790772998.313606:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790772998.313624:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790772998.313627:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790772998.313630:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790772998.313632:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1994 00000100:00080000:1.0:1790772998.313642:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790772998.313647:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790772998.313648:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790772998.313685:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534577536:192.168.204.150@tcp:8:kworker.0 00000008:00100000:1.0:1790772998.313687:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790772998.313713:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534578176/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790772998.313728:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534577664:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772998.313753:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772998.313762:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772998/real 1790772998] req@ffff9c4efac4f800 x1877759534577664/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772998.313771:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534577664/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772998.313778:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772998.313781:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772998.313784:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534577664:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772998.313789:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534577792:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772998.313800:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772998.313807:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772998/real 1790772998] req@ffff9c4efac4c700 x1877759534577792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772998.313815:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534577792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772998.313827:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772998.313829:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772998.313831:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534577792:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772998.313835:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534577920:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790772998.313846:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772998.313853:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772998/real 1790772998] req@ffff9c4efac4ed80 x1877759534577920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772998.313861:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534577920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772998.313867:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772998.313869:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772998.313871:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534577920:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790772998.313875:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534578048:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772998.313885:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772998.313892:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772998/real 1790772998] req@ffff9c4efac4dc00 x1877759534578048/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772998.313899:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534578048/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772998.313906:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772998.313908:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772998.313910:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534578048:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790772998.313923:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534578176:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790772998.313934:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790772998.313940:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790772998/real 1790772998] req@ffff9c4efac4df80 x1877759534578176/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790772998.313948:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534578176/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790772998.313955:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790772998.313958:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790772998.313960:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534578176:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790772998.376761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790772998.376761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772998.376762:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790772998.376770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772998.440737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772998.440748:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790772998.440852:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772998.568746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772998.568774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772998.632757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772998.696795:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772998.760716:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772998.760728:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772998.760756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772998.824716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772998.888722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772998.888726:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772998.888752:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772999.016722:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790772999.016728:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790772999.016744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772999.016763:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790772999.080721:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772999.080721:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790772999.080737:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790772999.080742:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790772999.080753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772999.080776:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790772999.208711:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790772999.272717:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790772999.336729:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790772999.400718:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790772999.400721:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790772999.400730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772999.400737:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790772999.464741:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790772999.464741:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790772999.464780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790772999.592712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790772999.592731:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790772999.656708:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790772999.720718:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772999.784739:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790772999.784739:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790772999.784739:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790772999.848716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790772999.912710:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790772999.912721:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790772999.912727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773000.040707:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773000.040726:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773000.040726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773000.040728:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773000.104718:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773000.104719:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773000.104722:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773000.104724:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773000.104755:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773000.104758:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773000.232715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773000.296719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773000.360749:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773000.424748:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773000.424748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773000.424748:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773000.424749:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773000.488740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773000.488742:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773000.488791:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773000.616741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773000.616741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773000.680746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773000.744708:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773000.808721:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773000.808721:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773000.808727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773000.872703:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773000.936705:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773000.936705:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773000.936719:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773001.064733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773001.064733:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773001.064734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773001.064735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773001.128726:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773001.128728:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773001.128758:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773001.128762:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773001.128806:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773001.128817:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773001.256702:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773001.320727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773001.384716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773001.448725:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773001.448725:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773001.448729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773001.448736:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773001.512713:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773001.512742:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773001.512759:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773001.640704:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773001.640708:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773001.704714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773001.768756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773001.832747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773001.832747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773001.832748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773001.896710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773001.960741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773001.960742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773001.960742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773002.088751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773002.088760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773002.088761:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773002.088765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773002.152720:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773002.152736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773002.152736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773002.152761:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773002.152777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773002.152778:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773002.280706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773002.344715:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773002.408711:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773002.472729:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773002.472729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773002.472729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773002.472769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773002.536764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773002.536784:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773002.536810:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773002.664722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773002.664723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773002.728759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773002.792721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773002.856720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773002.856720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773002.856728:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773002.920711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773002.984753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773002.984754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773002.984754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773003.112738:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773003.112739:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773003.112740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773003.112834:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773003.176727:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773003.176728:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773003.176735:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773003.176747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773003.176762:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773003.176776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773003.304719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773003.368730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773003.432718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773003.432752:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773003.432758:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773003.432763:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773003.432774:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 1999 00000100:00080000:0.0:1790773003.432785:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773003.432798:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773003.432845:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534578304/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773003.432871:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773003.432874:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773003.432876:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773003.432879:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1999 00000100:00080000:0.0:1790773003.432882:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773003.432886:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773003.432896:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773003.432906:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534578304:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773003.432910:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534578432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773003.432920:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773003.432923:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773003.432925:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773003.432929:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1999 00000100:00080000:0.0:1790773003.432931:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773003.432934:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773003.432938:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773003.432946:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534578560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773003.432958:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773003.432962:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773003.432963:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773003.432966:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1999 00000100:00080000:0.0:1790773003.432970:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773003.432981:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773003.432985:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773003.433010:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534578688/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790773003.433011:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773003.433021:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773003.433025:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773003.433027:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773003.433030:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 1999 00000100:00080000:0.0:1790773003.433034:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773003.433038:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773003.433043:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773003.433052:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534578816/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773003.433059:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773003.433062:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773003.433063:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773003.433066:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1999 00000100:00080000:0.0:1790773003.433070:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773003.433073:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773003/real 1790773003] req@ffff9c4ec7a7df80 x1877759534578304/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773003.433076:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773003.433083:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773003.433094:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534578944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773003.433102:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773003.433102:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534578304/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773003.433110:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773003.433113:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773003.433121:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1999 00000100:00080000:0.0:1790773003.433125:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773003.433125:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773003.433131:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773003.433134:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790773003.433137:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773003.433141:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534578304:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773003.433147:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534579072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773003.433154:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773003.433157:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773003.433158:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773003.433162:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1999 00000100:00080000:0.0:1790773003.433165:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773003.433168:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534578432:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773003.433169:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773003.433176:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773003.433187:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534579200/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773003.433200:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790773003.433203:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773003.433206:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773003.433209:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773003.433214:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 1999 00000100:00080000:0.0:1790773003.433218:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773003.433221:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773003/real 1790773003] req@ffff9c4ec7a7f480 x1877759534578432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773003.433224:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773003.433230:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773003.433239:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534578432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773003.433244:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534579328/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773003.433267:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773003.433272:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773003.433276:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534578432:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773003.433282:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534578560:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773003.433294:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773003.433302:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773003/real 1790773003] req@ffff9c4ec7a7c380 x1877759534578560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773003.433309:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534578560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773003.433317:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773003.433321:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773003.433324:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534578560:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773003.433331:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534578688:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773003.433352:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773003.433365:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773003/real 1790773003] req@ffff9c4ec7a7d880 x1877759534578688/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773003.433381:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534578688/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773003.433389:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773003.433392:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773003.433394:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534578688:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773003.433399:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534578816:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773003.433414:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773003.433421:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773003/real 1790773003] req@ffff9c4ec7a7ed80 x1877759534578816/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773003.433429:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534578816/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773003.433436:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773003.433438:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773003.433441:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534578816:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773003.433445:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534578944:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773003.433456:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773003.433462:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773003/real 1790773003] req@ffff9c4ec7a7ca80 x1877759534578944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773003.433469:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534578944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773003.433486:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773003.433488:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773003.433490:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534578944:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773003.433494:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534579072:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773003.433504:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773003.433510:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773003/real 1790773003] req@ffff9c4ec7a7e300 x1877759534579072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773003.433517:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534579072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773003.433523:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773003.433525:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773003.433527:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534579072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773003.433544:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534579200:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773003.433554:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773003.433561:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773003/real 1790773003] req@ffff9c4ec7a7fb80 x1877759534579200/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773003.433569:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534579200/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773003.433575:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773003.433577:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773003.433580:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534579200:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773003.433584:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534579328:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773003.433594:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773003.433600:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773003/real 1790773003] req@ffff9c4ec7a7c700 x1877759534579328/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773003.433607:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534579328/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773003.433614:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773003.433616:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773003.433618:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534579328:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773003.496756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773003.496763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773003.496764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773003.496765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773003.560744:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773003.560781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773003.560821:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773003.688733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773003.688748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773003.752740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773003.816748:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773003.880753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773003.880758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773003.880758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773003.944735:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773004.008722:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773004.008767:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773004.008768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773004.136744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773004.136744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773004.136745:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773004.136762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773004.200735:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773004.200735:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773004.200738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773004.200769:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773004.200774:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773004.200801:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773004.328714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773004.392734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773004.456706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773004.520736:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773004.520736:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773004.520737:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773004.520737:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773004.584741:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773004.584749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773004.584803:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773004.712720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773004.712720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773004.776720:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773004.840735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773004.904711:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773004.904772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773004.904777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773004.968715:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773005.032739:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773005.032739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773005.032747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773005.160726:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773005.160726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773005.160727:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773005.160748:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773005.224719:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773005.224725:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773005.224728:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773005.224761:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773005.224770:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773005.224772:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773005.352711:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773005.416752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773005.480751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773005.544737:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773005.544737:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773005.544737:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773005.544772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773005.608737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773005.608779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773005.608805:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773005.736735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773005.736735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773005.800735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773005.864728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773005.928738:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773005.928738:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773005.928781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773005.992719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773006.056713:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773006.056715:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773006.056757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773006.184740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773006.184740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773006.184740:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773006.184761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773006.248701:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773006.248718:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773006.248725:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773006.248760:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773006.248762:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773006.248763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773006.376717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773006.440702:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773006.505698:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773006.568733:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773006.568733:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773006.568734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773006.568737:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773006.632724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773006.632747:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773006.632828:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773006.760713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773006.760714:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773006.824755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773006.888708:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773006.952707:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773006.952716:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773006.952762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773007.016762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773007.080750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773007.080750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773007.080757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773007.208732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773007.208734:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773007.208759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773007.208760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773007.272730:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773007.272750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773007.272773:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773007.272792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773007.272795:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773007.272800:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773007.400704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773007.464774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773007.528724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773007.592731:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773007.592732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773007.592738:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773007.592741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773007.656724:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773007.656725:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773007.656789:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773007.784720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773007.784734:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773007.848738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773007.912731:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773007.976698:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773007.976703:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773007.976717:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773008.040725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773008.104725:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773008.104725:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773008.104734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773008.232730:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773008.232730:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773008.232733:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773008.232733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773008.296709:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773008.296716:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773008.296721:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773008.296727:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773008.296743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773008.296757:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773008.424725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773008.488707:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773008.552735:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773008.552754:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773008.552762:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773008.552767:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773008.552778:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2004 00000100:00080000:1.0:1790773008.552788:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773008.552801:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773008.552836:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534579456/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773008.552872:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773008.552875:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773008.552877:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773008.552882:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2004 00000100:00080000:1.0:1790773008.552886:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773008.552892:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773008.552916:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773008.552924:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534579456:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773008.552933:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534579584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773008.552944:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773008.552947:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773008.552949:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773008.552954:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2004 00000100:00080000:1.0:1790773008.552958:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773008.552964:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773008.552970:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773008.552981:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534579712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790773008.552991:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773008.553002:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773008.553006:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773008.553009:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773008.553014:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2004 00000100:00080000:1.0:1790773008.553019:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773008.553025:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773008.553030:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773008.553039:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773008/real 1790773008] req@ffff9c4efac4fb80 x1877759534579456/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773008.553046:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534579840/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773008.553083:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773008.553088:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773008.553091:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773008.553094:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534579456/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773008.553099:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2004 00000100:00080000:1.0:1790773008.553107:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773008.553114:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773008.553116:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773008.553123:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790773008.553124:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773008.553130:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534579456:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773008.553141:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534579968/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773008.553153:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773008.553157:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1790773008.553160:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534579584:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773008.553161:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773008.553171:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2004 00000100:00080000:1.0:1790773008.553177:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773008.553183:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773008.553189:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790773008.553197:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773008.553204:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534580096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773008.553212:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773008/real 1790773008] req@ffff9c4efac4df80 x1877759534579584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773008.553225:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534579584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773008.553231:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773008.553237:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773008.553240:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773008.553240:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773008.553248:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773008.553251:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2004 00000100:00100000:2.0:1790773008.553254:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534579584:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773008.553258:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773008.553263:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534579712:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773008.553269:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773008.553275:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790773008.553283:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773008.553291:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534580224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773008.553295:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773008/real 1790773008] req@ffff9c4efac4dc00 x1877759534579712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773008.553317:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773008.553320:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534579712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773008.553325:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773008.553331:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773008.553340:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2004 00000100:00080000:1.0:1790773008.553348:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773008.553352:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773008.553356:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773008.553362:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790773008.553366:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773008.553370:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534579712:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773008.553384:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534580352/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773008.553384:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534579840:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773008.553408:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790773008.553410:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773008.553415:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773008.553419:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773008.553423:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773008/real 1790773008] req@ffff9c4efac4ed80 x1877759534579840/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773008.553426:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2004 00000100:00080000:1.0:1790773008.553434:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773008.553440:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534579840/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773008.553444:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773008.553455:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:2.0:1790773008.553456:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773008.553463:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773008.553466:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534579840:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773008.553473:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534580480/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773008.553476:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534579968:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773008.553507:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773008.553522:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773008/real 1790773008] req@ffff9c4efac4c700 x1877759534579968/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773008.553536:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534579968/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773008.553543:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773008.553545:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773008.553548:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534579968:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773008.553565:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534580096:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773008.553581:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773008.553589:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773008/real 1790773008] req@ffff9c4efac4f800 x1877759534580096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773008.553597:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534580096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773008.553609:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773008.553611:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773008.553614:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534580096:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773008.553618:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534580224:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773008.553629:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773008.553635:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773008/real 1790773008] req@ffff9c4efac4f480 x1877759534580224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773008.553643:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534580224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773008.553649:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773008.553671:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773008.553675:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534580224:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773008.553683:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534580352:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773008.553705:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773008.553720:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773008/real 1790773008] req@ffff9c4efac4e680 x1877759534580352/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773008.553734:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534580352/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773008.553746:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773008.553748:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773008.553751:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534580352:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773008.553756:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534580480:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773008.553772:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773008.553780:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773008/real 1790773008] req@ffff9c4efac4ea00 x1877759534580480/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773008.553788:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534580480/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773008.553794:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773008.553796:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773008.553798:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534580480:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773008.616729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773008.616732:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773008.616733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773008.616773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773008.680710:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773008.680720:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773008.680840:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773008.808711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773008.808711:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773008.872752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773008.936739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773009.000708:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773009.000758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773009.000758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773009.064729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773009.128831:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773009.128831:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773009.128831:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773009.256744:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773009.256748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773009.256783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773009.256784:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773009.320730:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773009.320731:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773009.320732:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773009.320734:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773009.320767:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773009.320769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773009.448736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773009.512722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773009.576727:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773009.640716:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773009.640720:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773009.640722:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773009.640735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773009.704727:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773009.704763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773009.704795:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773009.832721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773009.832723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773009.896749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773009.960732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773010.024707:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773010.024718:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773010.024724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773010.088712:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773010.152735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773010.152735:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773010.152746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773010.280762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773010.280762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773010.280763:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773010.280763:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773010.344717:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773010.344719:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773010.344725:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773010.344743:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773010.344747:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773010.344766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773010.472742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773010.536785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773010.600741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773010.664744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773010.664756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773010.664756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773010.664761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773010.728785:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773010.728817:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773010.728835:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773010.856713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773010.856725:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773010.920728:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773010.984721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773011.048703:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773011.048718:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773011.048766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773011.112723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773011.176781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773011.176781:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773011.176797:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773011.304785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773011.304789:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773011.304799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773011.304803:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773011.368714:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773011.368782:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773011.368787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773011.368813:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773011.368834:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773011.368835:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773011.496708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773011.560727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773011.624712:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773011.688762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773011.688763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773011.688796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773011.688822:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773011.752766:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773011.752811:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773011.752863:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773011.880712:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773011.880733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773011.944724:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773012.008709:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773012.072704:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773012.072741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773012.072762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773012.136729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773012.200724:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773012.200774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773012.200798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773012.328794:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773012.328856:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773012.328886:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773012.328890:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773012.392772:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773012.392817:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773012.392833:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773012.392836:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773012.392842:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773012.392873:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773012.520716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773012.584706:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773012.648711:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773012.712793:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773012.712810:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773012.712894:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773012.712915:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773012.776883:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773012.776902:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773012.776953:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773012.904747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773012.904778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773012.968783:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773013.032793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773013.096773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773013.096829:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773013.096846:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773013.160741:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773013.224867:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773013.224871:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773013.224883:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773013.352771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773013.352856:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773013.352866:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773013.352866:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773013.416832:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773013.416892:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773013.416901:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773013.416901:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773013.416944:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773013.417010:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773013.544755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773013.608752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773013.672720:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773013.672720:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773013.672729:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773013.672735:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773013.672749:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2009 00000100:00080000:1.0:1790773013.672763:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773013.672779:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773013.672844:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534580608/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773013.672869:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773013.672874:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773013.672876:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773013.672879:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2009 00000100:00080000:1.0:1790773013.672883:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773013.672889:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773013.672900:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773013.672905:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534580608:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773013.672914:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534580736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773013.672923:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773013.672926:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773013.672927:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773013.672930:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2009 00000100:00080000:1.0:1790773013.672933:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773013.672936:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773013.672940:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773013.672950:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534580864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773013.672957:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773013.672960:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773013.672961:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773013.672964:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2009 00000100:00080000:1.0:1790773013.672975:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773013.672978:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773013.672982:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:2.0:1790773013.672989:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773013.672993:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534580992/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773013.673001:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773013.673004:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773013.673006:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773013.673009:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2009 00000100:00080000:1.0:1790773013.673011:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773013.673015:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773013.673018:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773013.673026:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534581120/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773013.673039:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773013.673043:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773013.673046:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773013.673046:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773013/real 1790773013] req@ffff9c4efac4e300 x1877759534580608/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773013.673055:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2009 00000100:00080000:1.0:1790773013.673061:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773013.673066:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773013.673070:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773013.673074:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534580608/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773013.673080:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534581248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773013.673090:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773013.673092:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773013.673096:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773013.673098:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773013.673100:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773013.673106:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534580608:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773013.673107:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2009 00000100:00080000:1.0:1790773013.673113:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773013.673118:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773013.673121:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773013.673128:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534581376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773013.673136:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534580736:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773013.673138:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773013.673144:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773013.673145:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773013.673150:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2009 00000100:00080000:1.0:1790773013.673153:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773013.673157:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773013.673162:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:2.0:1790773013.673166:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773013.673177:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534581504/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773013.673182:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773013/real 1790773013] req@ffff9c4efac4ea00 x1877759534580736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773013.673192:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773013.673199:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773013.673201:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773013.673205:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534580736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773013.673208:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2009 00000100:00080000:1.0:1790773013.673213:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773013.673219:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773013.673226:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773013.673236:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534581632/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773013.673238:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773013.673249:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773013.673255:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534580736:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773013.673264:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534580864:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773013.673291:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773013.673305:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773013/real 1790773013] req@ffff9c4efac4e680 x1877759534580864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773013.673319:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534580864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773013.673326:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773013.673329:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773013.673331:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534580864:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773013.673336:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534580992:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773013.673353:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773013.673360:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773013/real 1790773013] req@ffff9c4efac4f480 x1877759534580992/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773013.673369:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534580992/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773013.673376:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773013.673379:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773013.673381:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534580992:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773013.673385:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534581120:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773013.673396:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773013.673402:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773013/real 1790773013] req@ffff9c4efac4f800 x1877759534581120/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773013.673415:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534581120/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773013.673421:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773013.673424:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773013.673426:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534581120:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773013.673429:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534581248:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773013.673441:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773013.673447:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773013/real 1790773013] req@ffff9c4efac4c700 x1877759534581248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773013.673455:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534581248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773013.673466:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773013.673468:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773013.673470:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534581248:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773013.673487:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534581376:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773013.673498:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773013.673504:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773013/real 1790773013] req@ffff9c4efac4ed80 x1877759534581376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773013.673512:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534581376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773013.673522:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773013.673524:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773013.673526:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534581376:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773013.673530:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534581504:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773013.673540:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773013.673546:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773013/real 1790773013] req@ffff9c4efac4dc00 x1877759534581504/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773013.673554:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534581504/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773013.673561:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773013.673563:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773013.673565:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534581504:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773013.673578:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534581632:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773013.673599:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773013.673611:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773013/real 1790773013] req@ffff9c4efac4df80 x1877759534581632/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773013.673625:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534581632/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773013.673635:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773013.673637:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773013.673641:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534581632:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773013.736807:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773013.736807:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773013.736822:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773013.736829:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773013.800742:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773013.800776:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773013.800858:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773013.928713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773013.928742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773013.992749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773014.056768:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773014.120808:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773014.120816:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773014.120822:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773014.184727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773014.248718:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773014.248752:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773014.248759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773014.376750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773014.376751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773014.376800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773014.376801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773014.440736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773014.440738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773014.440769:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773014.440769:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773014.440814:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773014.440822:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773014.568736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773014.632731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773014.696709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773014.760741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773014.760742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773014.760796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773014.760799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773014.824705:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773014.824813:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773014.824953:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773014.952705:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773014.952728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773015.016758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773015.080792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773015.144711:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773015.144738:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773015.144745:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773015.208758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773015.272715:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773015.272791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773015.272798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773015.400731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773015.400736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773015.400797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773015.400800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773015.464744:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773015.464757:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773015.464780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773015.464780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773015.464819:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773015.464821:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773015.592829:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773015.656725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773015.720723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773015.784789:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773015.784789:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773015.784812:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773015.784817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773015.848767:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773015.848767:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773015.848832:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773015.976709:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773015.976731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773016.040786:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773016.104741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773016.168737:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773016.168737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773016.168763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773016.232719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773016.296779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773016.296781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773016.296798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773016.424721:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773016.424723:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773016.424758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773016.424778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773016.488733:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773016.488827:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773016.488827:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773016.488834:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773016.488836:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773016.488864:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773016.616716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773016.680753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773016.744710:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773016.808808:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773016.808849:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773016.808850:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773016.808853:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773016.872703:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773016.872814:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773016.872884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773017.000703:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773017.000713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773017.064752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773017.128775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773017.192711:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773017.192742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773017.192778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773017.256698:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773017.320752:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773017.320768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773017.320780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773017.448711:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773017.448744:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773017.448749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773017.448778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773017.512707:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773017.512761:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773017.512777:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773017.512792:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773017.512798:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773017.512824:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773017.640739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773017.704785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773017.768730:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773017.832758:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773017.832784:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773017.832791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773017.832797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773017.896776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773017.896786:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773017.896857:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773018.024756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773018.024761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773018.088750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773018.152761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773018.216754:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773018.216754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773018.216763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773018.280759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773018.344746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773018.344787:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773018.344791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773018.472745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773018.472746:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773018.472777:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773018.473697:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773018.536741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773018.536755:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773018.536778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773018.536787:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773018.536797:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773018.536804:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773018.664748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773018.728758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773018.792741:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773018.792745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773018.792755:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773018.792761:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773018.792773:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2014 00000100:00080000:1.0:1790773018.792786:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773018.792802:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773018.792848:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534581760/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773018.792882:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773018.792885:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773018.792887:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773018.792890:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2014 00000100:00080000:1.0:1790773018.792893:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773018.792898:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773018.792908:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773018.792920:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534581888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773018.792924:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534581760:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773018.792948:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773018.792951:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773018.792954:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773018.792960:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2014 00000100:00080000:1.0:1790773018.792963:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773018.792967:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773018.792973:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773018.792987:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534582016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773018.792996:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773018.792999:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773018.793000:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773018.793007:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2014 00000800:00000100:2.0:1790773018.793013:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773018.793015:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773018.793023:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773018.793028:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773018.793040:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534582144/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773018.793049:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773018.793053:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773018.793056:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773018.793061:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2014 00000100:00100000:2.0:1790773018.793063:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773018/real 1790773018] req@ffff9c4efac4fb80 x1877759534581760/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773018.793071:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773018.793089:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773018.793091:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534581760/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773018.793104:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:2.0:1790773018.793106:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773018.793115:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773018.793119:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534581760:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773018.793129:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534582272/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773018.793138:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773018.793142:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773018.793145:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773018.793150:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534581888:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773018.793155:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2014 00000100:00080000:1.0:1790773018.793165:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773018.793170:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773018.793176:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790773018.793185:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773018.793195:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534582400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773018.793202:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773018/real 1790773018] req@ffff9c4efac4df80 x1877759534581888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773018.793220:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773018.793229:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1790773018.793231:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534581888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773018.793235:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773018.793249:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2014 00000100:00080000:1.0:1790773018.793257:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773018.793260:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773018.793266:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773018.793268:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773018.793271:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534581888:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773018.793281:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534582016:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790773018.793282:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790773018.793304:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773018.793311:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534582528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773018.793320:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773018/real 1790773018] req@ffff9c4efac4dc00 x1877759534582016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773018.793356:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534582016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773018.793360:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773018.793374:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773018.793376:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773018.793377:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773018.793388:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773018.793391:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2014 00000100:00100000:2.0:1790773018.793398:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534582016:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773018.793399:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773018.793416:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773018.793417:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534582144:192.168.204.150@tcp:8:kworker.0 00000008:00100000:1.0:1790773018.793433:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:2.0:1790773018.793447:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773018.793452:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534582656/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773018.793459:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773018/real 1790773018] req@ffff9c4efac4ed80 x1877759534582144/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773018.793476:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773018.793492:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773018.793496:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773018.793498:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534582144/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773018.793509:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2014 00000100:00080000:1.0:1790773018.793517:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773018.793529:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773018.793539:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:2.0:1790773018.793557:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790773018.793563:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534582784/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773018.793566:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773018.793578:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534582144:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773018.793586:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534582272:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773018.793614:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773018.793626:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773018/real 1790773018] req@ffff9c4efac4c700 x1877759534582272/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773018.793635:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534582272/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773018.793641:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773018.793644:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773018.793646:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534582272:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773018.793686:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534582400:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773018.793723:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773018.793736:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773018/real 1790773018] req@ffff9c4efac4f800 x1877759534582400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773018.793748:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534582400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773018.793760:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773018.793762:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773018.793765:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534582400:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773018.793770:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534582528:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773018.793790:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773018.793802:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773018/real 1790773018] req@ffff9c4efac4f480 x1877759534582528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773018.793816:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534582528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773018.793826:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773018.793829:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773018.793832:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534582528:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773018.793838:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534582656:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773018.793856:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773018.793868:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773018/real 1790773018] req@ffff9c4efac4e680 x1877759534582656/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773018.793882:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534582656/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773018.793895:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773018.793899:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773018.793901:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534582656:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773018.793907:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534582784:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773018.793922:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773018.793931:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773018/real 1790773018] req@ffff9c4efac4ea00 x1877759534582784/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773018.793939:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534582784/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773018.793947:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773018.793949:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773018.793952:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534582784:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773018.856786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773018.856788:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773018.856791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773018.856793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773018.920753:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773018.920809:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773018.920919:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773019.048720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773019.048760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773019.112756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773019.176777:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773019.240762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773019.240765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773019.240775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773019.304732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773019.368748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773019.368750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773019.368759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773019.496739:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773019.496740:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773019.496749:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773019.496768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773019.560738:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773019.560738:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773019.560738:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773019.560746:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773019.560782:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773019.560785:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773019.688720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773019.752726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773019.816713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773019.880760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773019.880760:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773019.880761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773019.880764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773019.944717:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773019.944765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773019.944819:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773020.072741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773020.072743:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773020.136758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773020.200750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773020.264748:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773020.264748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773020.264750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773020.328725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773020.392706:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773020.392723:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773020.392744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773020.520738:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773020.520758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773020.520764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773020.520783:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773020.584726:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773020.584736:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773020.584740:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773020.584752:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773020.584767:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773020.584780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773020.712742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773020.776742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773020.840746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773020.904752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773020.904752:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773020.904753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773020.904758:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773020.968779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773020.968809:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773020.968813:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773021.096758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773021.096769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773021.160733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773021.224766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773021.288749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773021.288751:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773021.288751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773021.352713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773021.416725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773021.416731:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773021.416756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773021.544722:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773021.544784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773021.544784:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773021.544785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773021.608711:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773021.608741:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773021.608749:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773021.608773:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773021.608831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773021.608836:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773021.736752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773021.800730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773021.864722:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773021.928723:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773021.928725:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773021.928789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773021.928790:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773021.992741:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773021.992749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773021.992892:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773022.120714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773022.120720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773022.184760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773022.248733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773022.312712:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773022.312716:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773022.312752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773022.376732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773022.440750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773022.440751:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773022.440757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773022.568718:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773022.568746:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773022.568747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773022.568786:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773022.632722:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773022.632738:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773022.632752:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773022.632775:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773022.632777:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773022.632778:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773022.760702:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773022.824721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773022.888703:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773022.952764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773022.952803:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773022.952815:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773022.952826:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773023.016724:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773023.016741:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773023.016886:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773023.144722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773023.144723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773023.208754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773023.272715:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773023.336760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773023.336760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773023.336770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773023.400706:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773023.464704:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773023.464728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773023.464778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773023.592707:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773023.592733:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773023.592740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773023.592787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773023.656728:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773023.656743:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773023.656765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773023.656777:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773023.656783:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773023.656798:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773023.784698:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773023.848715:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773023.912739:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773023.912739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773023.912757:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773023.912762:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773023.912775:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2019 00000100:00080000:1.0:1790773023.912790:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773023.912806:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773023.912853:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534582912/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773023.912882:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773023.912887:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773023.912889:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773023.912894:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2019 00000100:00080000:1.0:1790773023.912898:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773023.912905:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773023.912917:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773023.912925:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534582912:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773023.912932:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534583040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773023.912943:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773023.912946:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773023.912948:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773023.912953:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2019 00000100:00080000:1.0:1790773023.912956:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773023.912962:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773023.912967:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773023.912977:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534583168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773023.912986:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773023.912990:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773023.912992:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773023.912996:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2019 00000100:00080000:1.0:1790773023.912999:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773023.913004:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790773023.913006:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773023.913009:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773023.913019:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534583296/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773023.913028:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773023.913031:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773023.913033:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773023.913036:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2019 00000100:00080000:1.0:1790773023.913040:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773023.913044:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773023.913049:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773023.913059:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534583424/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773023.913060:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773023/real 1790773023] req@ffff9c4efac4e300 x1877759534582912/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773023.913070:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773023.913074:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773023.913076:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773023.913079:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534582912/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773023.913080:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2019 00000100:00080000:1.0:1790773023.913084:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773023.913089:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773023.913092:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773023.913094:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790773023.913098:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773023.913104:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534583552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773023.913104:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534582912:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773023.913113:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773023.913117:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773023.913119:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773023.913123:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2019 00000100:00080000:1.0:1790773023.913127:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773023.913129:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534583040:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773023.913132:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773023.913136:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773023.913146:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534583680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790773023.913152:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773023.913157:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773023.913161:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773023.913164:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773023.913167:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773023/real 1790773023] req@ffff9c4efac4ea00 x1877759534583040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773023.913169:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2019 00000100:00080000:1.0:1790773023.913173:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773023.913178:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773023.913180:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534583040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773023.913183:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773023.913193:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534583808/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773023.913200:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773023.913202:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773023.913204:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773023.913206:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773023.913208:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773023.913208:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534583040:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773023.913213:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2019 00000100:00100000:2.0:1790773023.913215:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534583168:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773023.913217:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773023.913222:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773023.913226:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:2.0:1790773023.913234:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773023.913237:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534583936/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773023.913245:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773023/real 1790773023] req@ffff9c4efac4e680 x1877759534583168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773023.913257:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534583168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773023.913267:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773023.913271:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773023.913275:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534583168:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773023.913282:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534583296:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773023.913303:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773023.913313:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773023/real 1790773023] req@ffff9c4efac4f480 x1877759534583296/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773023.913322:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534583296/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773023.913330:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773023.913332:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773023.913335:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534583296:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773023.913339:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534583424:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773023.913350:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773023.913357:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773023/real 1790773023] req@ffff9c4efac4f800 x1877759534583424/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773023.913365:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534583424/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773023.913371:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773023.913373:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773023.913376:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534583424:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773023.913379:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534583552:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773023.913390:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773023.913396:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773023/real 1790773023] req@ffff9c4efac4c700 x1877759534583552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773023.913404:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534583552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773023.913416:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773023.913418:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773023.913421:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534583552:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773023.913438:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534583680:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773023.913448:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773023.913455:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773023/real 1790773023] req@ffff9c4efac4ed80 x1877759534583680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773023.913463:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534583680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773023.913469:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773023.913471:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773023.913473:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534583680:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773023.913477:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534583808:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773023.913487:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773023.913493:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773023/real 1790773023] req@ffff9c4efac4dc00 x1877759534583808/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773023.913501:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534583808/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773023.913508:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773023.913510:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773023.913513:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534583808:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773023.913516:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534583936:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773023.913526:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773023.913532:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773023/real 1790773023] req@ffff9c4efac4df80 x1877759534583936/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773023.913539:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534583936/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773023.913545:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773023.913547:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773023.913549:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534583936:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773023.976735:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773023.976740:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773023.976767:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773023.976768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773024.040746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773024.040759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773024.040915:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773024.168721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773024.168731:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773024.232741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773024.296783:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773024.360730:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773024.360756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773024.360756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773024.424708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773024.488711:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773024.488740:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773024.488749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773024.616731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773024.616731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773024.616737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773024.616768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773024.680719:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773024.680725:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773024.680744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773024.680766:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773024.680784:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773024.680786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773024.808706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773024.872709:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773024.936739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773025.000719:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773025.000729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773025.000730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773025.000767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773025.064776:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773025.064779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773025.064803:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773025.192727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773025.192787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773025.256727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773025.320716:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773025.384721:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773025.384723:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773025.384760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773025.448709:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773025.512714:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773025.512741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773025.512749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773025.640821:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773025.640822:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773025.640827:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773025.640827:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773025.704741:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773025.704741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773025.704763:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773025.704773:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773025.704810:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773025.704832:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773025.832727:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773025.896711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773025.960742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773026.024731:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773026.024731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773026.024758:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773026.024760:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773026.088719:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773026.088755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:3.0:1790773026.088945:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773026.216721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773026.216740:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773026.280758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773026.344748:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773026.408698:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773026.408731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773026.408731:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773026.472711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773026.536768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773026.536768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773026.536768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773026.664729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773026.664729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773026.664740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773026.664747:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773026.728713:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773026.728719:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773026.728742:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773026.728751:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773026.728767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773026.728781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773026.856705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773026.920719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773026.984732:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773027.048748:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773027.048785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773027.048794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773027.048795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773027.112750:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773027.112762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773027.112942:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773027.240722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773027.240733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773027.304803:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773027.368733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773027.432697:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773027.432764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773027.432765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773027.496708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773027.560723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773027.560748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773027.560748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773027.688743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773027.688750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773027.688772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773027.688788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773027.752739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773027.752751:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773027.752752:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773027.752780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773027.752812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773027.752814:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773027.880707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773027.944723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773028.008750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773028.072726:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773028.072735:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773028.072735:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773028.072779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773028.136755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773028.136772:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773028.136906:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773028.264720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773028.264753:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773028.328749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773028.392720:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773028.456693:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773028.456749:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773028.456782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773028.520759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773028.584758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773028.584759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773028.584771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773028.712746:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773028.712746:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773028.712765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773028.712809:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773028.776710:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773028.776757:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773028.776805:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773028.776828:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773028.776834:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773028.776842:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773028.904709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773028.968709:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773029.032741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773029.032756:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773029.032765:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773029.032770:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773029.032780:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2024 00000100:00080000:0.0:1790773029.032788:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773029.032799:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773029.032833:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534584064/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773029.032857:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773029.032860:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773029.032862:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773029.032866:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2024 00000100:00080000:0.0:1790773029.032869:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773029.032873:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773029.032881:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773029.032893:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534584192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773029.032902:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534584064:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773029.032915:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773029.032919:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773029.032921:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773029.032926:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2024 00000100:00080000:0.0:1790773029.032930:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773029.032935:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773029.032939:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773029.032948:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534584320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773029.032956:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773029.032959:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773029.032960:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773029.032964:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2024 00000100:00080000:0.0:1790773029.032967:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773029.032971:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773029.032975:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773029.032983:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534584448/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773029.032989:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790773029.032996:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773029.033001:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773029.033005:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773029.033021:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2024 00000100:00080000:0.0:1790773029.033026:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773029.033030:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773029.033035:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773029.033046:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534584576/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773029.033054:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773029.033056:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773029.033058:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773029.033061:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2024 00000100:00080000:0.0:1790773029.033066:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773029.033069:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773029.033076:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773029.033086:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773029/real 1790773029] req@ffff9c4ec7a7d180 x1877759534584064/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773029.033087:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534584704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773029.033159:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773029.033164:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773029.033167:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773029.033175:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2024 00000100:00100000:2.0:1790773029.033177:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534584064/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773029.033179:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773029.033193:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773029.033209:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790773029.033220:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773029.033227:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534584832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773029.033229:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773029.033247:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534584064:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773029.033254:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773029.033260:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773029.033262:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773029.033267:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2024 00000100:00080000:0.0:1790773029.033272:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773029.033276:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773029.033280:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773029.033289:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534584192:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773029.033296:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534584960/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773029.033324:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773029.033329:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773029.033331:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773029.033335:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2024 00000100:00080000:0.0:1790773029.033340:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773029.033348:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773029.033353:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:2.0:1790773029.033364:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773029.033383:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534585088/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773029.033397:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773029/real 1790773029] req@ffff9c4ec7a7c700 x1877759534584192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773029.033410:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534584192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773029.033430:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773029.033433:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773029.033436:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534584192:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773029.033441:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534584320:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773029.033455:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773029.033462:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773029/real 1790773029] req@ffff9c4ec7a7fb80 x1877759534584320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773029.033469:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534584320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773029.033476:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773029.033478:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773029.033480:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534584320:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773029.033485:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534584448:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773029.033517:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773029.033525:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773029/real 1790773029] req@ffff9c4ec7a7e300 x1877759534584448/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773029.033533:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534584448/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773029.033541:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773029.033543:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773029.033546:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534584448:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773029.033550:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534584576:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773029.033563:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773029.033569:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773029/real 1790773029] req@ffff9c4ec7a7ca80 x1877759534584576/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773029.033577:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534584576/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773029.033584:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773029.033586:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773029.033588:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534584576:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773029.033592:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534584704:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773029.033603:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773029.033610:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773029/real 1790773029] req@ffff9c4ec7a7ed80 x1877759534584704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773029.033618:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534584704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773029.033630:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773029.033632:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773029.033634:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534584704:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773029.033639:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534584832:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773029.033668:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773029.033677:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773029/real 1790773029] req@ffff9c4ec7a7d880 x1877759534584832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773029.033685:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534584832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773029.033692:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773029.033694:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773029.033697:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534584832:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773029.033716:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534584960:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773029.033729:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773029.033736:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773029/real 1790773029] req@ffff9c4ec7a7c380 x1877759534584960/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773029.033744:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534584960/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773029.033756:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773029.033759:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773029.033761:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534584960:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773029.033765:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534585088:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773029.033776:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773029.033782:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773029/real 1790773029] req@ffff9c4ec7a7f480 x1877759534585088/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773029.033789:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534585088/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773029.033796:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773029.033806:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773029.033809:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534585088:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773029.096717:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773029.096734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773029.096757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773029.096757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773029.160740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773029.160806:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773029.160837:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773029.288778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773029.288808:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773029.352748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773029.416737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773029.480696:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773029.480730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773029.480769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773029.544708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773029.608763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773029.608766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773029.608810:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773029.736795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773029.736795:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773029.736822:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773029.736855:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773029.800731:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773029.800738:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773029.800744:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773029.800766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773029.800785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773029.800810:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773029.928712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773029.992750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773030.056766:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773030.120779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773030.120785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773030.120787:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773030.120811:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773030.184762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773030.184796:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773030.184799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773030.312738:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773030.312767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773030.376748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773030.440735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773030.504699:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773030.504737:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773030.504765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773030.568743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773030.632753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773030.632753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773030.632776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773030.760796:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773030.760796:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773030.760824:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773030.760828:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773030.824785:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773030.824795:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773030.824801:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773030.824801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773030.824821:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773030.824840:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773030.952719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773031.016713:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773031.080755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773031.144759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773031.144759:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773031.144759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773031.144769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773031.208718:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773031.208777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773031.208794:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773031.336714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773031.336750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773031.400709:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773031.464741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773031.528750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773031.528772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773031.528773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773031.592733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773031.656746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773031.656746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773031.656758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773031.784756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773031.784757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773031.784757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773031.784768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773031.848743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773031.848774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773031.848775:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773031.848779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773031.848806:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773031.848809:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773031.976710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773032.040747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773032.104715:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773032.168725:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773032.168727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773032.168730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773032.168736:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773032.232764:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773032.232785:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773032.232804:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773032.360701:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773032.360743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773032.424752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773032.488728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773032.552729:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773032.552729:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773032.552759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773032.616710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773032.680754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773032.680754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773032.680763:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773032.808765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773032.808765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773032.808765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773032.808770:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773032.872744:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773032.872756:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773032.872769:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773032.872792:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773032.872796:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773032.872798:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773033.001697:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773033.064743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773033.128751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773033.192731:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773033.192731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773033.192771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773033.192773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773033.256718:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773033.256778:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773033.256814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773033.384724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773033.384753:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773033.448715:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773033.512733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773033.576821:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773033.576821:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773033.576827:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773033.640732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773033.704733:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773033.704750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773033.704775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773033.832772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773033.832775:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773033.832778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773033.832779:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773033.896736:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773033.896736:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773033.896736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773033.896770:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773033.896778:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773033.896805:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773034.024710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773034.088732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773034.152744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773034.152777:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773034.152782:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773034.152785:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773034.152796:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2030 00000100:00080000:2.0:1790773034.152807:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773034.152820:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773034.152874:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534585216/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773034.152905:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773034.152908:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773034.152910:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773034.152913:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2030 00000100:00080000:2.0:1790773034.152916:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773034.152919:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773034.152929:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773034.152940:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534585344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773034.152949:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773034.152952:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773034.152954:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773034.152960:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2030 00000100:00080000:2.0:1790773034.152965:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773034.152966:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534585216:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773034.152972:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773034.152987:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773034.153001:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534585472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773034.153008:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773034.153012:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773034.153014:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773034.153018:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2030 00000100:00080000:2.0:1790773034.153021:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773034.153028:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773034.153033:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773034.153040:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534585600/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773034.153048:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773034.153051:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773034.153059:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773034.153063:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773034.153072:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2030 00000100:00080000:2.0:1790773034.153078:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773034.153086:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773034.153094:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773034.153096:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773034/real 1790773034] req@ffff9c4efbcc0700 x1877759534585216/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773034.153116:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534585728/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773034.153118:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534585216/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773034.153178:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773034.153179:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773034.153187:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773034.153191:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773034.153196:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773034.153200:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2030 00000100:00080000:2.0:1790773034.153206:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773034.153210:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534585216:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773034.153218:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773034.153226:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773034.153237:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534585856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773034.153243:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534585344:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773034.153261:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773034.153265:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773034.153267:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773034.153271:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2030 00000100:00080000:2.0:1790773034.153273:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773034.153276:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773034.153280:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773034.153293:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534585984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773034.153300:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773034.153308:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773034.153316:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773034.153318:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773034.153329:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2030 00000100:00100000:0.0:1790773034.153330:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773034/real 1790773034] req@ffff9c4efbcc0a80 x1877759534585344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773034.153334:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773034.153349:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773034.153355:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534585344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773034.153355:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773034.153374:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534586112/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773034.153380:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773034.153383:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773034.153388:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773034.153392:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773034.153409:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773034.153416:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534585344:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773034.153422:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2030 00000100:00080000:2.0:1790773034.153430:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773034.153432:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534585472:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773034.153438:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773034.153448:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773034.153463:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534586240/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773034.153474:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773034.153491:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773034/real 1790773034] req@ffff9c4efbcc2680 x1877759534585472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773034.153500:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534585472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773034.153508:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773034.153510:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773034.153513:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534585472:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773034.153519:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534585600:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773034.153534:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773034.153545:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773034/real 1790773034] req@ffff9c4efbcc2300 x1877759534585600/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773034.153558:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534585600/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773034.153569:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773034.153573:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773034.153578:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534585600:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773034.153585:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534585728:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773034.153605:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773034.153613:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773034/real 1790773034] req@ffff9c4efbcc1880 x1877759534585728/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773034.153622:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534585728/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773034.153629:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773034.153631:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773034.153634:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534585728:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773034.153673:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534585856:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773034.153687:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773034.153696:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773034/real 1790773034] req@ffff9c4efbcc1500 x1877759534585856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773034.153705:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534585856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773034.153717:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773034.153720:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773034.153722:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534585856:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773034.153727:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534585984:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773034.153737:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773034.153744:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773034/real 1790773034] req@ffff9c4efbcc1f80 x1877759534585984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773034.153752:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534585984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773034.153758:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773034.153760:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773034.153762:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534585984:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773034.153766:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534586112:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773034.153776:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773034.153783:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773034/real 1790773034] req@ffff9c4efbcc1c00 x1877759534586112/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773034.153791:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534586112/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773034.153803:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773034.153805:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773034.153807:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534586112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773034.153811:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534586240:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773034.153821:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773034.153829:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773034/real 1790773034] req@ffff9c4efbcc2a00 x1877759534586240/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773034.153840:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534586240/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773034.153851:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773034.153855:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773034.153859:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534586240:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773034.216742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773034.216768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773034.216798:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773034.216803:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773034.280796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773034.280846:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773034.280878:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773034.408718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773034.408739:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773034.472747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773034.536715:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773034.600731:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773034.600732:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773034.600778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773034.664712:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773034.728724:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773034.728724:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773034.728759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773034.856736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773034.856738:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773034.856739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773034.856775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773034.920715:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773034.920733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773034.920759:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773034.920766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773034.920767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773034.920777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773035.048719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773035.112718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773035.176720:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773035.240733:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773035.240736:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773035.240738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773035.240741:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773035.304715:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773035.304739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773035.304770:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773035.432718:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773035.432742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773035.496718:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773035.560730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773035.624795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773035.624795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773035.624796:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773035.688720:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773035.752718:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773035.752750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773035.752750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773035.880782:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773035.880782:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773035.880853:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773035.880856:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773035.944784:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773035.944788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773035.944792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773035.944799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773035.944848:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773035.944853:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773036.072732:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773036.136744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773036.200707:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773036.264731:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773036.264731:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773036.264731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773036.264768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773036.328715:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773036.328782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773036.328847:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773036.456730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773036.456738:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773036.520746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773036.584772:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773036.648711:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773036.648789:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773036.648791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773036.712710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773036.776731:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773036.776731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773036.776741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773036.904715:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773036.904723:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773036.904781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773036.904784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773036.968733:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773036.968733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773036.968738:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773036.968778:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773036.968786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773036.968789:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773037.096709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773037.160700:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773037.224734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773037.288742:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773037.288743:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773037.288772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773037.288776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773037.352735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773037.352736:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773037.352840:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773037.480723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773037.480730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773037.544751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773037.608714:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773037.672741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773037.672741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773037.672759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773037.736705:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773037.800715:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773037.800720:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773037.800768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773037.928739:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773037.928740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773037.928743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773037.928790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773037.992761:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773037.992765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773037.992768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773037.992807:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773037.992818:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773037.992822:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773038.120724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773038.184764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773038.248726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773038.312728:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773038.312738:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773038.312741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773038.312769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773038.376731:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773038.376779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773038.376909:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773038.504719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773038.504720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773038.568783:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773038.632731:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773038.696715:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773038.696720:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773038.696776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773038.760716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773038.824733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773038.824734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773038.824779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773038.952715:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773038.952725:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773038.952740:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773038.952783:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773039.016743:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773039.016763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773039.016792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773039.016795:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773039.016798:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773039.016822:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773039.144718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773039.208713:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773039.272742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773039.272755:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773039.272763:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773039.272768:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773039.272778:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2035 00000100:00080000:0.0:1790773039.272788:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773039.272800:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773039.272839:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534586368/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773039.272871:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773039.272876:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773039.272878:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773039.272884:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2035 00000100:00080000:0.0:1790773039.272889:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773039.272895:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773039.272901:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534586368:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773039.272908:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773039.272927:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534586496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773039.272940:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773039.272945:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773039.272948:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773039.272953:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2035 00000100:00080000:0.0:1790773039.272958:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790773039.272964:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773039.272966:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773039.272973:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773039.272984:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534586624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773039.272991:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773039.272994:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773039.272995:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773039.272998:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2035 00000100:00080000:0.0:1790773039.273002:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773039.273006:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773039/real 1790773039] req@ffff9c4ec7a7df80 x1877759534586368/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773039.273009:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773039.273016:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773039.273029:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534586752/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773039.273033:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534586368/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773039.273046:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773039.273051:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773039.273052:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773039.273053:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773039.273061:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773039.273062:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2035 00000100:00080000:0.0:1790773039.273068:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773039.273070:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534586368:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773039.273073:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773039.273081:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773039.273090:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534586880/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773039.273098:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773039.273100:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534586496:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773039.273105:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773039.273109:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773039.273115:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2035 00000100:00080000:0.0:1790773039.273120:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790773039.273123:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773039.273130:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773039.273137:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773039.273138:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773039/real 1790773039] req@ffff9c4ec7a7f480 x1877759534586496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773039.273150:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534586496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773039.273157:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534587008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773039.273166:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773039.273174:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773039.273181:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534586496:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773039.273187:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773039.273189:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534586624:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773039.273195:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773039.273200:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773039.273206:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2035 00000800:00000100:2.0:1790773039.273209:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773039.273213:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773039.273220:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773039/real 1790773039] req@ffff9c4ec7a7c380 x1877759534586624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773039.273223:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773039.273235:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773039.273236:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534586624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773039.273246:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773039.273249:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773039.273254:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534587136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773039.273255:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534586624:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773039.273264:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534586752:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773039.273279:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773039.273284:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1790773039.273286:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773039.273288:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773039.273310:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2035 00000100:00100000:2.0:1790773039.273314:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773039/real 1790773039] req@ffff9c4ec7a7d880 x1877759534586752/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773039.273318:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773039.273328:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534586752/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773039.273330:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773039.273339:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:2.0:1790773039.273342:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773039.273347:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773039.273350:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534586752:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773039.273357:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534587264/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773039.273365:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534586880:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773039.273374:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773039.273379:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773039.273381:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790773039.273386:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773039.273390:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2035 00000100:00080000:0.0:1790773039.273396:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773039.273399:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773039/real 1790773039] req@ffff9c4ec7a7ed80 x1877759534586880/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773039.273405:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773039.273415:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773039.273416:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534586880/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773039.273427:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773039.273433:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773039.273434:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534587392/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773039.273437:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534586880:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773039.273449:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534587008:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773039.273468:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773039.273477:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773039/real 1790773039] req@ffff9c4ec7a7ca80 x1877759534587008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773039.273485:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534587008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773039.273497:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773039.273499:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773039.273501:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534587008:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773039.273505:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534587136:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773039.273515:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773039.273521:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773039/real 1790773039] req@ffff9c4ec7a7e300 x1877759534587136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773039.273529:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534587136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773039.273535:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773039.273536:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773039.273539:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534587136:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773039.273547:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534587264:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773039.273558:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773039.273565:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773039/real 1790773039] req@ffff9c4ec7a7fb80 x1877759534587264/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773039.273572:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534587264/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773039.273580:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773039.273582:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773039.273584:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534587264:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773039.273587:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534587392:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773039.273597:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773039.273604:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773039/real 1790773039] req@ffff9c4ec7a7c700 x1877759534587392/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773039.273612:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534587392/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773039.273619:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773039.273621:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773039.273623:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534587392:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773039.336753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773039.336770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773039.336775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773039.336787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773039.400734:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773039.400777:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773039.400887:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773039.528704:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773039.528707:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773039.592747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773039.656744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773039.720719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773039.720719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773039.720746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773039.784713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773039.848824:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773039.848826:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773039.848827:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773039.976748:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773039.976791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773039.976791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773039.976792:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773040.040721:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773040.040744:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773040.040749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773040.040756:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773040.040772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773040.040803:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773040.168722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773040.232713:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773040.296698:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773040.360721:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773040.360721:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773040.360724:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773040.360724:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773040.424733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773040.424733:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773040.424806:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773040.552709:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773040.552722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773040.616710:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773040.680708:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773040.744767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773040.744767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773040.744767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773040.808727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773040.872734:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773040.872734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773040.872735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773041.000700:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773041.000729:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773041.000731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773041.000731:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773041.064716:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773041.064740:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773041.064743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773041.064762:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773041.064764:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773041.064792:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773041.192699:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773041.256739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773041.320713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773041.384772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773041.384786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773041.384809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773041.384814:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773041.448706:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773041.448866:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773041.448967:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773041.576744:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773041.576751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773041.640762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773041.704782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773041.768745:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773041.768753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773041.768755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773041.832742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773041.896738:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773041.896739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773041.896749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773042.024736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773042.024738:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773042.024739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773042.024752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773042.088750:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773042.088750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773042.088818:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773042.088819:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773042.088819:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773042.088820:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773042.216720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773042.280750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773042.344737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773042.408733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773042.408741:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773042.408744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773042.408744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773042.472735:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773042.472735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773042.472787:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773042.600719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773042.600726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773042.664734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773042.728731:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773042.792732:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773042.792732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773042.792733:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773042.856725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773042.920746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773042.920749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773042.920761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773043.048720:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773043.048725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773043.048727:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773043.048731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773043.112746:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773043.112754:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773043.112760:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773043.112786:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773043.112803:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773043.112816:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773043.240706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773043.304733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773043.368701:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773043.432729:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773043.432729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773043.432729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773043.432732:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773043.496741:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773043.496750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773043.496780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773043.624715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773043.624730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773043.688725:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773043.752739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773043.816718:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773043.816735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773043.816735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773043.880753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773043.944752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773043.944753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773043.944753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773044.072713:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773044.072715:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773044.072736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773044.072736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773044.136708:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773044.136719:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773044.136728:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773044.136742:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773044.136752:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773044.136763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773044.264713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773044.328733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773044.392718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773044.392774:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773044.392780:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773044.392784:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773044.392792:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2040 00000100:00080000:1.0:1790773044.392800:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773044.392810:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773044.392845:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534587520/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773044.392867:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773044.392870:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773044.392872:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773044.392875:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2040 00000100:00080000:1.0:1790773044.392877:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773044.392880:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773044.392888:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773044.392900:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534587648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773044.392912:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773044.392913:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534587520:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773044.392919:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773044.392923:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773044.392929:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2040 00000100:00080000:1.0:1790773044.392933:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773044.392938:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773044.392944:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773044.392959:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534587776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773044.392971:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773044.392974:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773044.392976:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790773044.392981:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773044.392983:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2040 00000100:00080000:1.0:1790773044.392988:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773044.392994:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773044.392998:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773044.393008:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534587904/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773044.393015:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773044.393018:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773044.393019:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773044.393024:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2040 00000100:00080000:1.0:1790773044.393026:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773044.393031:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773044.393031:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773044/real 1790773044] req@ffff9c4efac4fb80 x1877759534587520/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773044.393039:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773044.393052:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534588032/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773044.393057:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534587520/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773044.393065:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773044.393072:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773044.393074:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773044.393075:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773044.393084:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773044.393085:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2040 00000100:00080000:1.0:1790773044.393090:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773044.393093:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534587520:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773044.393097:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773044.393105:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773044.393114:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534588160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773044.393120:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773044.393123:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1790773044.393125:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534587648:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773044.393126:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773044.393135:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2040 00000100:00080000:1.0:1790773044.393139:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773044.393142:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773044.393146:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790773044.393154:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773044.393158:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534588288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773044.393167:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773044.393170:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773044/real 1790773044] req@ffff9c4efac4df80 x1877759534587648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773044.393171:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773044.393174:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773044.393182:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2040 00000100:00080000:1.0:1790773044.393187:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773044.393191:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534587648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773044.393210:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773044.393218:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:2.0:1790773044.393229:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790773044.393231:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534588416/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773044.393237:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773044.393244:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534587648:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773044.393257:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773044.393257:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534587776:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773044.393266:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773044.393269:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773044.393273:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2040 00000100:00080000:1.0:1790773044.393276:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773044.393281:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773044.393285:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:2.0:1790773044.393287:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773044.393298:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534588544/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773044.393305:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773044/real 1790773044] req@ffff9c4efac4dc00 x1877759534587776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773044.393323:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534587776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773044.393333:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773044.393338:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773044.393341:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534587776:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773044.393349:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534587904:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773044.393373:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773044.393387:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773044/real 1790773044] req@ffff9c4efac4ed80 x1877759534587904/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773044.393400:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534587904/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773044.393411:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773044.393415:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773044.393418:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534587904:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773044.393424:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534588032:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773044.393446:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773044.393459:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773044/real 1790773044] req@ffff9c4efac4c700 x1877759534588032/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773044.393471:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534588032/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773044.393481:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773044.393485:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773044.393489:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534588032:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773044.393496:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534588160:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773044.393519:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773044.393532:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773044/real 1790773044] req@ffff9c4efac4f800 x1877759534588160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773044.393546:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534588160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773044.393570:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773044.393574:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773044.393577:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534588160:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773044.393599:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534588288:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773044.393620:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773044.393632:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773044/real 1790773044] req@ffff9c4efac4f480 x1877759534588288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773044.393644:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534588288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773044.393674:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773044.393678:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773044.393681:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534588288:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773044.393687:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534588416:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773044.393709:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773044.393721:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773044/real 1790773044] req@ffff9c4efac4e680 x1877759534588416/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773044.393733:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534588416/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773044.393743:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773044.393746:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773044.393751:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534588416:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773044.393757:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534588544:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773044.393775:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773044.393785:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773044/real 1790773044] req@ffff9c4efac4ea00 x1877759534588544/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773044.393798:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534588544/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773044.393808:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773044.393811:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773044.393815:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534588544:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773044.456709:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773044.456729:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773044.456730:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773044.456735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773044.520738:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773044.520740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773044.520780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773044.648710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773044.648713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773044.712720:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773044.776734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773044.840745:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773044.840745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773044.840745:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773044.904722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773044.968710:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773044.968736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773044.968744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773045.096732:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773045.096738:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773045.096738:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773045.096740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773045.160728:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773045.160730:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773045.160732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773045.160737:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773045.160760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773045.160765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773045.288710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773045.352737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773045.416707:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773045.480745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773045.480746:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773045.480746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773045.480746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773045.544720:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773045.544748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773045.544828:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773045.672723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773045.672764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773045.736741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773045.800804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773045.864734:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773045.864737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773045.864743:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773045.928716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773045.992749:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773045.992749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773045.992751:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773046.120767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773046.120768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773046.120775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773046.120789:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773046.184721:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773046.184721:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773046.184749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773046.184750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773046.184767:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773046.184767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773046.312731:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773046.376746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773046.440713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773046.504714:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773046.504718:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773046.504729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773046.504741:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773046.568729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773046.568744:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773046.568758:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773046.696720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773046.696735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773046.760718:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773046.824722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773046.888715:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773046.888720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773046.888723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773046.952706:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773047.016706:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773047.016718:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773047.016725:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773047.144717:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773047.144719:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773047.144730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773047.144743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773047.208706:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773047.208725:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773047.208728:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773047.208741:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773047.208756:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773047.208761:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773047.336714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773047.400718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773047.464704:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773047.528707:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773047.528729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773047.528730:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773047.528735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773047.592751:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773047.592753:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773047.592843:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773047.720710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773047.720730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773047.784766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773047.848734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773047.912734:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773047.912735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773047.912735:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773047.976732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773048.040739:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773048.040740:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773048.040740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773048.168711:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773048.168718:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773048.168726:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773048.168726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773048.232721:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773048.232723:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773048.232738:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773048.232766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773048.232772:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773048.232777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773048.360722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773048.424720:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773048.488710:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773048.552729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773048.552731:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773048.552731:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773048.552732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773048.616704:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773048.616710:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773048.616794:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773048.744715:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773048.744715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773048.808733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773048.872717:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773048.936762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773048.936762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773048.936763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773049.000730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773049.064821:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773049.064824:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773049.064826:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773049.192753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773049.192753:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773049.192754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773049.192754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773049.256719:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773049.256734:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773049.256735:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773049.256739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773049.256763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773049.256772:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773049.384706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773049.448728:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773049.512728:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773049.512743:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773049.512751:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773049.512756:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773049.512766:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2045 00000100:00080000:1.0:1790773049.512776:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773049.512788:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773049.512828:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534588672/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773049.512852:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773049.512855:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773049.512856:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773049.512860:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2045 00000100:00080000:1.0:1790773049.512863:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773049.512868:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773049.512877:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773049.512888:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534588672:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773049.512891:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534588800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773049.512902:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773049.512904:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773049.512906:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773049.512909:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2045 00000100:00080000:1.0:1790773049.512912:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773049.512915:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773049.512918:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773049.512926:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534588928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773049.512933:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773049.512936:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773049.512937:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773049.512941:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2045 00000100:00080000:1.0:1790773049.512944:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773049.512948:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773049.512952:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:2.0:1790773049.512954:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773049.512962:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534589056/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773049.512970:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773049.512973:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773049.512975:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773049.512978:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2045 00000100:00080000:1.0:1790773049.512980:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773049.512984:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773049.512988:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773049.512998:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773049/real 1790773049] req@ffff9c4efac4e300 x1877759534588672/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773049.512999:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534589184/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773049.513027:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773049.513031:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773049.513033:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773049.513036:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2045 00000100:00100000:2.0:1790773049.513040:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534588672/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773049.513041:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773049.513048:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773049.513055:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790773049.513060:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773049.513066:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773049.513070:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534588672:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773049.513073:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534589312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773049.513081:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773049.513084:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773049.513085:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773049.513089:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2045 00000100:00080000:1.0:1790773049.513092:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773049.513112:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534588800:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773049.513134:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773049.513147:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773049/real 1790773049] req@ffff9c4efac4ea00 x1877759534588800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773049.513157:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534588800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773049.513170:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773049.513173:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773049.513175:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534588800:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773049.513180:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534588928:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773049.513191:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773049.513197:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773049/real 1790773049] req@ffff9c4efac4e680 x1877759534588928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773049.513205:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534588928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773049.513211:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773049.513213:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773049.513215:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534588928:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773049.513219:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534589056:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773049.513226:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773049.513231:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790773049.513236:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773049.513246:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534589440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773049.513247:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773049/real 1790773049] req@ffff9c4efac4f480 x1877759534589056/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773049.513271:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773049.513276:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534589056/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773049.513277:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773049.513281:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773049.513288:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2045 00000100:00080000:1.0:1790773049.513294:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773049.513295:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773049.513299:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773049.513300:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790773049.513306:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773049.513306:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534589056:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773049.513316:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534589184:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773049.513319:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534589568/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773049.513328:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773049.513332:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773049.513334:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790773049.513338:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773049.513342:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2045 00000100:00080000:1.0:1790773049.513348:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773049.513352:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773049/real 1790773049] req@ffff9c4efac4f800 x1877759534589184/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773049.513353:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773049.513361:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773049.513371:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534589184/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773049.513376:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534589696/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773049.513394:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773049.513400:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773049.513404:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534589184:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773049.513409:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534589312:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773049.513423:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773049.513430:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773049/real 1790773049] req@ffff9c4efac4c700 x1877759534589312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773049.513438:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534589312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773049.513449:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773049.513451:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773049.513453:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534589312:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773049.513467:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534589440:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773049.513478:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773049.513484:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773049/real 1790773049] req@ffff9c4efac4ed80 x1877759534589440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773049.513492:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534589440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773049.513497:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773049.513499:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773049.513502:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534589440:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773049.513506:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534589568:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773049.513516:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773049.513521:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773049/real 1790773049] req@ffff9c4efac4dc00 x1877759534589568/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773049.513529:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534589568/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773049.513541:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773049.513543:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773049.513546:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534589568:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773049.513552:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534589696:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773049.513570:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773049.513582:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773049/real 1790773049] req@ffff9c4efac4df80 x1877759534589696/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773049.513596:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534589696/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773049.513607:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773049.513610:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773049.513613:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534589696:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773049.576743:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773049.576744:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773049.576744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773049.576749:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773049.640731:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773049.640731:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773049.640790:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773049.768717:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773049.768724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773049.832779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773049.896717:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773049.960724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773049.960726:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773049.960735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773050.024725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773050.088723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773050.088726:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773050.088730:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773050.216707:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773050.216720:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773050.216730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773050.216732:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773050.280723:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773050.280731:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773050.280731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773050.280749:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773050.280770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773050.280779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773050.408724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773050.472721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773050.536701:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773050.600722:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773050.600730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773050.600731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773050.600737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773050.664713:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773050.664729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773050.664778:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773050.792737:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773050.792737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773050.856716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773050.920724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773050.984737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773050.984738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773050.984738:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773051.048811:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773051.112726:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773051.112726:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773051.112727:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773051.240713:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773051.240715:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773051.240731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773051.240732:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773051.304709:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773051.304718:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773051.304735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773051.304739:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773051.304752:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773051.304755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773051.432751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773051.496714:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773051.560772:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773051.624744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773051.624746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773051.624746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773051.624784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773051.688736:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773051.688763:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773051.688937:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773051.816750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773051.816770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773051.880763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773051.944815:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773052.008804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773052.008805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773052.008807:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773052.072720:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773052.136742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773052.136742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773052.136762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773052.264818:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773052.264824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773052.264824:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773052.264839:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773052.328803:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773052.328803:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773052.328828:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773052.328840:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773052.328880:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773052.328905:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773052.456759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773052.520762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773052.584756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773052.648748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773052.648749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773052.648827:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773052.648835:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773052.712775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773052.712782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773052.712953:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773052.840773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773052.840779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773052.904778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773052.968768:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773053.032767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773053.032782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773053.032782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773053.096718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773053.160726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773053.160769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773053.160794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773053.288756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773053.288756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773053.288756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773053.288794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773053.352752:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773053.352767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773053.352773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773053.352776:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773053.352817:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773053.352821:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773053.480733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773053.544750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773053.608729:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773053.672722:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773053.672761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773053.672767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773053.672771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773053.736764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773053.736780:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773053.736784:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773053.864711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773053.864711:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773053.928747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773053.992770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773054.056742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773054.056789:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773054.056792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773054.120702:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773054.184729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773054.184739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773054.184740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773054.312752:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773054.312786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773054.312787:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773054.312789:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773054.376733:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773054.376745:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773054.376749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773054.376763:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773054.376783:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773054.376804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773054.504728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773054.568725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773054.632733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773054.632745:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773054.632751:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773054.632754:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773054.632763:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2050 00000100:00080000:3.0:1790773054.632771:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773054.632781:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773054.632816:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534589824/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773054.632842:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773054.632846:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773054.632847:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773054.632851:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2050 00000100:00080000:3.0:1790773054.632853:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773054.632856:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773054.632865:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773054.632875:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534589952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773054.632890:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534589824:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773054.632890:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773054.632898:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773054.632902:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773054.632906:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2050 00000100:00080000:3.0:1790773054.632908:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773054.632911:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773054.632915:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773054.632923:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534590080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773054.632930:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773054.632933:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773054.632935:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773054.632939:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2050 00000100:00080000:3.0:1790773054.632943:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773054.632948:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773054.632954:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773054.632966:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534590208/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773054.632978:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773054.632993:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773054.633008:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773054.633013:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534589952:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773054.633015:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2050 00000100:00080000:3.0:1790773054.633022:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773054.633027:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773054.633033:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773054.633047:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534590336/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773054.633057:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773054.633062:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1790773054.633063:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534590080:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773054.633065:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773054.633074:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2050 00000100:00080000:3.0:1790773054.633077:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773054.633082:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773054.633087:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773054.633092:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534590208:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773054.633098:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534590464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773054.633106:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773054.633109:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773054.633111:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773054.633115:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2050 00000100:00080000:3.0:1790773054.633118:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773054.633119:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534590336:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790773054.633124:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773054.633131:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773054.633143:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534590592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773054.633146:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534590464:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773054.633164:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773054.633167:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773054.633169:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773054.633174:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2050 00000100:00100000:2.0:1790773054.633177:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534590592:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773054.633179:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773054.633187:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773054.633191:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773054.633203:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534590720/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773054.633217:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773054.633220:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773054.633223:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773054.633230:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2050 00000100:00080000:3.0:1790773054.633233:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773054.633236:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534590720:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790773054.633240:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773054.633246:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773054.633257:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534590848/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773054.633277:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534590848:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773054.696769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773054.696773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773054.696773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773054.696773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773054.760743:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773054.760798:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773054.760878:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773054.888706:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773054.888715:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773054.952752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773055.016737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773055.080758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773055.080768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773055.080771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773055.144748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773055.208726:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773055.208726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773055.208750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773055.336786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773055.336794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773055.336800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773055.336802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773055.400710:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773055.400729:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773055.400731:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773055.400774:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773055.400781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773055.400784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773055.528723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773055.592740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773055.656710:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773055.720723:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773055.720725:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773055.720731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773055.720732:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773055.784726:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773055.784727:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773055.784780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773055.912736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773055.912738:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773055.976707:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773056.040725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773056.104725:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773056.104727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773056.104734:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773056.168706:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773056.232710:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773056.232710:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773056.232731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773056.360716:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773056.360717:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773056.360731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773056.360733:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773056.424728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773056.424728:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773056.424728:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773056.424736:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773056.424760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773056.424775:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773056.552715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773056.616706:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773056.680716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773056.744745:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773056.744745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773056.744746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773056.744746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773056.808730:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773056.808735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773056.808812:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773056.936711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773056.936711:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773057.000731:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773057.064743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773057.128732:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773057.128732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773057.128732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773057.192709:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773057.256713:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773057.256717:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773057.256718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773057.384753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773057.384753:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773057.384753:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773057.384753:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773057.448719:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773057.448729:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773057.448732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773057.448742:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773057.448763:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773057.448768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773057.576699:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773057.640726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773057.704725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:1.0:1790773057.704881:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:1.0:1790773057.704920:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1790773057.704952:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773057.704999:0:2384:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:1.0:1790773057.705013:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773057.705022:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790773057.705026:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773054/real 1790773057] req@ffff9c4ef73c1c00 x1877759534590848/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790773057.705031:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773057.705040:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773057.705049:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790773057.705050:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534590848/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790773057.705058:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:2.0:1790773057.705066:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000800:00000100:1.0:1790773057.705067:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:2.0:1790773057.705080:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773057.705086:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534590848:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773057.705095:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773054/real 1790773057] req@ffff9c4ef73c0000 x1877759534590720/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773057.705110:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534590720/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773057.705121:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773057.705125:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773057.705129:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534590720:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773057.705145:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773054/real 1790773057] req@ffff9c4ef73c2300 x1877759534590592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773057.705160:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534590592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773057.705169:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773057.705173:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773057.705182:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534590592:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773057.705189:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773054/real 1790773057] req@ffff9c4ef73c2a00 x1877759534590336/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773057.705204:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534590336/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773057.705214:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773057.705218:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773057.705222:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534590336:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773057.705229:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773054/real 1790773057] req@ffff9c4ef73c0700 x1877759534590464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773057.705241:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534590464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773057.705256:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773057.705260:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773057.705263:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534590464:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773057.705270:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773054/real 1790773057] req@ffff9c4ef73c1500 x1877759534589952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773057.705282:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534589952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773057.705298:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773057.705303:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773057.705308:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534589952:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773057.705315:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773054/real 1790773057] req@ffff9c4ef73c0a80 x1877759534590080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773057.705327:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534590080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773057.705337:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773057.705342:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773057.705346:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534590080:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773057.705353:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773054/real 1790773057] req@ffff9c4ef73c3800 x1877759534590208/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773057.705365:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534590208/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773057.705375:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773057.705379:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773057.705383:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534590208:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773057.705391:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773054/real 1790773057] req@ffff9c4ef73c2d80 x1877759534589824/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773057.705404:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534589824/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773057.705413:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773057.705417:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773057.705421:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534589824:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773057.768715:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773057.768720:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773057.768728:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773057.768728:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773057.832716:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773057.832738:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773057.832786:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773057.960711:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773057.960758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773058.024784:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773058.088719:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773058.152712:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773058.152720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773058.152736:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773058.216704:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773058.280725:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773058.280728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773058.280730:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773058.408710:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773058.408725:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773058.408727:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773058.408777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773058.472702:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773058.472713:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773058.472721:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773058.472732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773058.472748:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773058.472759:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773058.600717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773058.664706:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773058.728714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773058.792749:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773058.792749:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773058.792749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773058.792749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773058.856729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773058.856734:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773058.856803:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773058.984719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773058.984722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773059.048712:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773059.112721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773059.176716:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773059.176737:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773059.176737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773059.240706:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773059.304728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773059.304729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773059.304734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773059.432729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773059.432729:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773059.432729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773059.432729:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773059.496708:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773059.496714:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773059.496737:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773059.496737:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773059.496747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773059.496752:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773059.624726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773059.688746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773059.752716:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773059.752725:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1790773059.752727:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773059.752730:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773059.752739:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2055 00000100:00080000:3.0:1790773059.752748:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773059.752759:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773059.752790:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534590976/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773059.752811:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773059.752814:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773059.752816:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773059.752819:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2055 00000100:00080000:3.0:1790773059.752822:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773059.752826:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773059.752834:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773059.752843:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534590976:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773059.752847:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534591104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773059.752857:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773059.752860:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773059.752862:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773059.752867:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2055 00000100:00080000:3.0:1790773059.752879:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773059.752894:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773059.752913:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790773059.752917:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773059.752932:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534591232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773059.752944:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773059.752949:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773059.752951:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773059.752956:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2055 00000100:00080000:3.0:1790773059.752962:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773059.752964:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773059/real 1790773059] req@ffff9c4ef73c0e00 x1877759534590976/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773059.752971:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773059.752981:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773059.752994:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534590976/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773059.752997:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534591360/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773059.753028:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773059.753029:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773059.753035:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773059.753041:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773059.753041:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773059.753052:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534590976:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773059.753052:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2055 00000100:00080000:3.0:1790773059.753061:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773059.753069:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773059.753075:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773059.753084:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534591104:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773059.753090:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534591488/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773059.753110:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773059.753115:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:2.0:1790773059.753119:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773059.753119:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773059.753128:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2055 00000100:00080000:3.0:1790773059.753134:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773059.753141:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773059/real 1790773059] req@ffff9c4ef73c2d80 x1877759534591104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773059.753143:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773059.753152:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773059.753163:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534591104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773059.753169:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534591616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773059.753187:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773059.753191:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773059.753193:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773059.753194:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773059.753201:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2055 00000100:00080000:3.0:1790773059.753205:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773059.753210:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773059.753214:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773059.753220:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534591104:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790773059.753223:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773059.753233:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534591232:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773059.753240:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534591744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773059.753252:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773059.753256:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773059.753259:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790773059.753263:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773059.753267:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2055 00000100:00080000:3.0:1790773059.753272:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773059.753280:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773059.753288:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773059.753289:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773059/real 1790773059] req@ffff9c4ef73c3800 x1877759534591232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773059.753304:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534591872/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773059.753309:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534591232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773059.753326:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773059.753330:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773059.753330:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773059.753336:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773059.753339:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773059.753345:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2055 00000100:00100000:2.0:1790773059.753349:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534591232:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773059.753352:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773059.753358:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534591360:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790773059.753361:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773059.753370:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773059.753385:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534592000/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790773059.753396:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773059.753413:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773059/real 1790773059] req@ffff9c4ef73c0a80 x1877759534591360/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773059.753428:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534591360/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773059.753440:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773059.753446:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773059.753450:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534591360:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773059.753471:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534591488:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773059.753494:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773059.753507:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773059/real 1790773059] req@ffff9c4ef73c1500 x1877759534591488/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773059.753521:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534591488/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773059.753539:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773059.753543:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773059.753548:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534591488:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773059.753555:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534591616:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773059.753579:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773059.753594:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773059/real 1790773059] req@ffff9c4ef73c0700 x1877759534591616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773059.753616:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534591616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773059.753633:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773059.753637:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773059.753642:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534591616:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773059.753649:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534591744:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773059.753697:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773059.753731:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773059/real 1790773059] req@ffff9c4ef73c2a00 x1877759534591744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773059.753747:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534591744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773059.753755:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773059.753759:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773059.753762:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534591744:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773059.753769:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534591872:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773059.753792:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773059.753806:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773059/real 1790773059] req@ffff9c4ef73c2300 x1877759534591872/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773059.753818:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534591872/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773059.753838:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773059.753843:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773059.753846:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534591872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773059.753852:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534592000:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773059.753873:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773059.753883:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773059/real 1790773059] req@ffff9c4ef73c0000 x1877759534592000/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773059.753909:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534592000/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773059.753924:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773059.753928:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773059.753931:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534592000:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773059.816726:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773059.816727:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773059.816727:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773059.816730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773059.880728:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773059.880746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773059.880771:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773060.008719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773060.008721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773060.072755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773060.136762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773060.200764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773060.200764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773060.200783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773060.264705:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773060.328741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773060.328743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773060.328743:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773060.456713:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773060.456739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773060.456743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773060.456747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773060.520724:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773060.520727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773060.520754:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773060.520768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773060.520776:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773060.520791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773060.648726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773060.712727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773060.776738:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773060.840741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773060.840741:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773060.840741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773060.840742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773060.904729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773060.904747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773060.904788:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773061.032724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773061.032726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773061.096731:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773061.160713:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773061.224725:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773061.224729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773061.224734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773061.288736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773061.352698:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773061.352716:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773061.352722:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773061.480739:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773061.480739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773061.480739:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773061.480757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773061.544717:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773061.544733:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773061.544733:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773061.544739:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773061.544759:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773061.544762:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773061.672704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773061.736738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773061.800731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773061.864744:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773061.864744:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773061.864744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773061.864744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773061.928758:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773061.928782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773061.928801:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773062.056718:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773062.056718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773062.120732:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773062.184743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773062.248785:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773062.248786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773062.248786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773062.312725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773062.376737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773062.376737:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773062.376744:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773062.504716:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773062.504726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773062.504728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773062.504746:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773062.568717:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773062.568721:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773062.568733:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773062.568740:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773062.568754:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773062.568760:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773062.696724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773062.760753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773062.824716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773062.888733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773062.888733:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773062.888733:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773062.888760:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773062.952748:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773062.952748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773062.952857:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773063.080716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773063.080719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773063.144729:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773063.208731:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773063.272720:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773063.272720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773063.272746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773063.336715:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773063.400810:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773063.400810:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773063.400817:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773063.528736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773063.528736:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773063.528740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773063.528744:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773063.592727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773063.592742:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773063.592743:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773063.592784:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773063.592811:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773063.592837:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773063.720721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773063.784733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773063.848709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773063.912721:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773063.912721:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773063.912727:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773063.912731:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773063.976712:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773063.976732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773063.976808:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773064.104715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773064.104724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773064.168732:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773064.232722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773064.296734:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773064.296734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773064.296735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773064.360751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773064.424722:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773064.424723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773064.424735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773064.552717:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773064.552730:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773064.552735:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773064.552735:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773064.616732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773064.616733:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773064.616744:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773064.616756:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773064.616772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773064.616772:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773064.744711:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773064.808726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773064.872709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773064.872717:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773064.872723:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773064.872727:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773064.872736:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2060 00000100:00080000:3.0:1790773064.872745:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773064.872755:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773064.872789:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534592128/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773064.872833:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773064.872837:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773064.872840:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773064.872845:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2060 00000100:00080000:3.0:1790773064.872850:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773064.872857:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773064.872865:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534592128:192.168.204.150@tcp:250:kworker.0 00000008:00100000:3.0:1790773064.872866:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773064.872883:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534592256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773064.872893:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773064.872895:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773064.872897:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773064.872900:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2060 00000100:00080000:3.0:1790773064.872914:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773064.872919:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773064.872922:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790773064.872928:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773064.872934:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534592384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773064.872946:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773064.872951:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773064.872953:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773064.872958:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2060 00000100:00080000:3.0:1790773064.872964:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773064.872969:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773064/real 1790773064] req@ffff9c4ef73c1c00 x1877759534592128/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773064.872973:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773064.872983:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773064.872998:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534592512/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773064.873000:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534592128/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773064.873014:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773064.873020:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773064.873024:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773064.873028:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2060 00000100:00080000:2.0:1790773064.873029:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773064.873032:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773064.873040:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773064.873040:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773064.873049:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773064.873052:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534592128:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773064.873059:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534592640/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773064.873066:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773064.873069:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773064.873071:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773064.873073:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2060 00000100:00080000:3.0:1790773064.873075:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773064.873079:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773064.873084:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534592256:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790773064.873086:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773064.873098:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534592768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773064.873107:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773064.873110:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773064.873112:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790773064.873115:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773064.873119:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2060 00000100:00080000:3.0:1790773064.873124:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773064.873129:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773064.873132:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773064.873133:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773064/real 1790773064] req@ffff9c4ef73c0000 x1877759534592256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773064.873144:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534592896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773064.873149:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534592256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773064.873155:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773064.873162:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773064.873165:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773064.873170:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2060 00000100:00080000:3.0:1790773064.873174:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773064.873178:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773064.873179:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773064.873187:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:2.0:1790773064.873188:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773064.873194:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534592256:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773064.873199:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534593024/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773064.873204:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534592384:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773064.873215:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773064.873218:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773064.873220:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773064.873224:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2060 00000100:00080000:3.0:1790773064.873227:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790773064.873229:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773064.873233:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773064.873238:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773064.873243:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773064/real 1790773064] req@ffff9c4ef73c2300 x1877759534592384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773064.873250:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534593152/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773064.873259:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534592384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773064.873272:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773064.873275:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773064.873277:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534592384:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773064.873282:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534592512:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773064.873294:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773064.873300:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773064/real 1790773064] req@ffff9c4ef73c2a00 x1877759534592512/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773064.873308:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534592512/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773064.873315:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773064.873317:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773064.873319:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534592512:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773064.873323:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534592640:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773064.873333:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773064.873339:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773064/real 1790773064] req@ffff9c4ef73c0700 x1877759534592640/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773064.873346:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534592640/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773064.873352:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773064.873354:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773064.873357:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534592640:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773064.873378:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534592768:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773064.873390:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773064.873397:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773064/real 1790773064] req@ffff9c4ef73c1500 x1877759534592768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773064.873404:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534592768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773064.873417:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773064.873419:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773064.873422:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534592768:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773064.873426:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534592896:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773064.873437:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773064.873443:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773064/real 1790773064] req@ffff9c4ef73c0a80 x1877759534592896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773064.873451:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534592896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773064.873456:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773064.873459:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773064.873461:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534592896:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773064.873464:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534593024:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773064.873474:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773064.873481:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773064/real 1790773064] req@ffff9c4ef73c3800 x1877759534593024/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773064.873489:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534593024/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773064.873496:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773064.873499:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773064.873501:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534593024:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773064.873504:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534593152:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773064.873514:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773064.873520:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773064/real 1790773064] req@ffff9c4ef73c2d80 x1877759534593152/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773064.873528:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534593152/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773064.873535:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773064.873537:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773064.873539:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534593152:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773064.936738:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773064.936738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773064.936742:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773064.936742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773065.000713:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773065.000757:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773065.000808:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773065.128719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773065.128731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773065.192750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773065.256747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773065.320722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773065.320771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773065.320771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773065.384710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773065.448806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773065.448807:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773065.448807:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773065.576738:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773065.576738:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773065.576741:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773065.576745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773065.640733:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773065.640757:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773065.640798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773065.640801:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773065.640822:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773065.640831:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773065.768727:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773065.832714:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773065.896714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773065.960728:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773065.960728:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773065.960728:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773065.960730:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773066.024715:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773066.024729:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773066.024790:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773066.152731:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773066.152731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773066.216728:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773066.280730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773066.344717:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773066.344727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773066.344727:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773066.408708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773066.472728:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773066.472728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773066.472728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773066.600745:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773066.600746:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773066.600746:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773066.600746:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773066.664730:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773066.664741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773066.664741:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773066.664749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773066.664784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773066.664790:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773066.792726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773066.856715:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773066.920721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773066.984768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773066.984768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773066.984805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773066.984805:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773067.048789:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773067.048806:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773067.048817:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773067.176778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773067.176784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773067.240742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773067.304739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773067.368722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773067.368763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773067.368765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773067.432743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773067.496739:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773067.496775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773067.496790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773067.624719:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773067.624719:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773067.624741:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773067.624759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773067.688737:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773067.688759:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773067.688760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773067.688769:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773067.688796:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773067.688807:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773067.816714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773067.880721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773067.944714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773068.008766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773068.008781:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773068.008785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773068.008786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773068.072732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773068.072791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773068.072868:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773068.200777:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773068.200779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773068.264779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773068.328771:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773068.392762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773068.392762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773068.392777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773068.456712:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773068.520760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773068.520766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773068.520770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773068.648777:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773068.648778:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773068.648779:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773068.648789:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773068.712737:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773068.712747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773068.712769:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773068.712792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773068.712795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773068.712815:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773068.840733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773068.904704:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773068.968739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773069.032741:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773069.032741:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773069.032749:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773069.032749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773069.096737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773069.096774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773069.096796:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773069.224732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773069.224739:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773069.288775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773069.352749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773069.416738:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773069.416745:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773069.416746:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773069.480730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773069.544764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773069.544764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773069.544764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773069.672719:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773069.672731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773069.672733:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773069.672737:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773069.736734:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773069.736749:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773069.736754:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773069.736764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773069.736780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773069.736782:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773069.864754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773069.928730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773069.992746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773069.992752:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773069.992760:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773069.992765:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773069.992776:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2065 00000100:00080000:0.0:1790773069.992787:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773069.992798:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773069.992836:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534593280/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773069.992864:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773069.992870:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773069.992872:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773069.992878:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2065 00000100:00080000:0.0:1790773069.992883:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773069.992890:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773069.992896:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534593280:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773069.992902:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773069.992921:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534593408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773069.992934:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773069.992939:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773069.992942:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773069.992947:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2065 00000100:00080000:0.0:1790773069.992952:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790773069.992956:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773069.992962:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773069.992969:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773069.992986:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534593536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773069.992997:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773069/real 1790773069] req@ffff9c4ec7a7d180 x1877759534593280/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773069.992998:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773069.993010:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773069.993015:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773069.993021:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2065 00000100:00100000:2.0:1790773069.993025:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534593280/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773069.993028:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773069.993039:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773069.993041:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773069.993046:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773069.993050:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534593280:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773069.993051:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773069.993068:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534593664/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773069.993073:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534593408:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773069.993092:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773069.993097:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1790773069.993098:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773069.993102:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773069.993109:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2065 00000100:00100000:2.0:1790773069.993112:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773069/real 1790773069] req@ffff9c4ec7a7c700 x1877759534593408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773069.993115:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773069.993126:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773069.993129:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534593408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773069.993135:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:2.0:1790773069.993146:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773069.993152:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534593792/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773069.993152:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773069.993158:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534593408:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773069.993168:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534593536:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773069.993175:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773069.993180:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773069.993183:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790773069.993186:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773069.993193:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2065 00000100:00100000:2.0:1790773069.993198:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773069/real 1790773069] req@ffff9c4ec7a7fb80 x1877759534593536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773069.993199:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773069.993209:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773069.993213:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534593536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773069.993217:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790773069.993221:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773069.993225:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773069.993227:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534593536:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773069.993235:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534593920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773069.993240:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534593664:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773069.993258:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790773069.993263:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773069.993265:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773069.993270:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773069.993274:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773069/real 1790773069] req@ffff9c4ec7a7e300 x1877759534593664/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773069.993279:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2065 00000100:00080000:0.0:1790773069.993288:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773069.993290:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534593664/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773069.993298:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773069.993303:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790773069.993307:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790773069.993308:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773069.993313:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534593664:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773069.993319:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534593792:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773069.993326:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534594048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790773069.993335:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773069.993345:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773069.993345:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773069/real 1790773069] req@ffff9c4ec7a7ca80 x1877759534593792/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773069.993355:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773069.993360:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773069.993361:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534593792/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773069.993371:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2065 00000100:00080000:0.0:1790773069.993379:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773069.993382:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773069.993387:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773069.993390:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773069.993394:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534593792:192.168.204.150@tcp:8:kworker.0 00000008:00100000:0.0:1790773069.993400:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773069.993406:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534593920:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773069.993420:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534594176/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790773069.993424:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773069.993434:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773069/real 1790773069] req@ffff9c4ec7a7ed80 x1877759534593920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773069.993439:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773069.993445:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1790773069.993446:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534593920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773069.993450:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773069.993462:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2065 00000100:00080000:2.0:1790773069.993467:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773069.993468:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773069.993475:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773069.993479:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773069.993481:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534593920:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773069.993490:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534594048:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773069.993491:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:2.0:1790773069.993508:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773069.993510:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534594304/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773069.993519:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773069/real 1790773069] req@ffff9c4ec7a7d880 x1877759534594048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773069.993530:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534594048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773069.993536:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773069.993539:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773069.993543:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534594048:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773069.993552:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534594176:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773069.993563:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773069.993570:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773069/real 1790773069] req@ffff9c4ec7a7c380 x1877759534594176/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773069.993577:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534594176/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773069.993585:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773069.993586:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773069.993589:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534594176:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773069.993593:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534594304:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773069.993602:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773069.993608:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773069/real 1790773069] req@ffff9c4ec7a7f480 x1877759534594304/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773069.993615:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534594304/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773069.993621:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773069.993623:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773069.993626:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534594304:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773070.056742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773070.056750:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773070.056752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773070.056755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773070.120755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773070.120755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773070.120785:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773070.248751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773070.248752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773070.312730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773070.376748:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773070.440778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773070.440778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773070.440903:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773070.504718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773070.568745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773070.568745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773070.568750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773070.696754:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773070.696755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773070.696760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773070.696763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773070.760735:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773070.760738:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773070.760757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773070.760761:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773070.760784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773070.760784:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773070.888718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773070.952748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773071.016750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773071.080730:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773071.080732:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773071.080739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773071.080743:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773071.144720:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773071.144760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773071.144787:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773071.272745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773071.272758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773071.336751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773071.400750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773071.464748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773071.464749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773071.464751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773071.528744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773071.592769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773071.592769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773071.592769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773071.720726:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773071.720736:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773071.720737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773071.720742:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773071.784730:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773071.784740:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773071.784746:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773071.784748:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773071.784767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773071.784771:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773071.912724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773071.976736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773072.040732:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773072.104732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773072.104742:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773072.104751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773072.104753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773072.168711:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773072.168756:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773072.168804:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773072.296727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773072.296732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773072.360739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773072.424740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773072.488746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773072.488746:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773072.488747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773072.552725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773072.616742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773072.616742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773072.616744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773072.744719:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773072.744730:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773072.744742:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773072.744742:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773072.808724:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773072.808726:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773072.808726:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773072.808732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773072.808756:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773072.808771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773072.936703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773073.000714:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773073.064721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773073.128742:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773073.128745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773073.128745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773073.128746:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773073.192750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773073.192763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773073.192805:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773073.320733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773073.320733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773073.384733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773073.448725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773073.512701:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773073.512715:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773073.512723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773073.576695:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773073.640729:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773073.640730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773073.640730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773073.768721:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773073.768721:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773073.768721:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773073.768725:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773073.832735:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773073.832735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773073.832735:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773073.832736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773073.832766:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773073.832777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773073.960700:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773074.024706:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773074.088723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773074.152747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773074.152747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773074.152748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773074.152749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773074.216729:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773074.216750:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773074.216773:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773074.344716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773074.344716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773074.408716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773074.472714:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773074.536710:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773074.536710:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773074.536717:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773074.600699:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773074.664725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773074.664729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773074.664730:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773074.792727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773074.792728:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773074.792728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773074.792736:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773074.856728:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773074.856734:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773074.856745:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773074.856745:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773074.856773:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773074.856782:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773074.984740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773075.048743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773075.112725:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773075.112726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773075.112734:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773075.112739:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773075.112749:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2071 00000100:00080000:2.0:1790773075.112760:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773075.112773:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773075.112813:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534594432/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773075.112843:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773075.112848:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773075.112850:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773075.112855:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2071 00000100:00080000:2.0:1790773075.112861:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773075.112867:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773075.112878:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773075.112881:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534594432:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773075.112898:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534594560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773075.112918:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773075.112928:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773075.112935:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773075.112940:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2071 00000100:00080000:2.0:1790773075.112945:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773075.112950:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773075.112957:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790773075.112958:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773075.112977:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534594688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773075.112989:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773075.112993:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773075.112996:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773075.113000:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773075/real 1790773075] req@ffff9c4efbcc0380 x1877759534594432/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773075.113005:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2071 00000100:00080000:2.0:1790773075.113014:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773075.113021:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773075.113025:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534594432/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773075.113028:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00080000:0.0:1790773075.113040:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1790773075.113045:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534594816/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773075.113046:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773075.113054:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534594432:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773075.113063:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773075.113066:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773075.113067:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773075.113071:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2071 00000100:00080000:2.0:1790773075.113073:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773075.113077:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773075.113081:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534594560:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790773075.113085:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773075.113104:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534594944/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773075.113108:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773075.113121:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773075/real 1790773075] req@ffff9c4efbcc3100 x1877759534594560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773075.113122:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773075.113133:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773075.113138:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773075.113143:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2071 00000100:00080000:2.0:1790773075.113149:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773075.113157:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534594560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773075.113158:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773075.113178:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773075.113182:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773075.113190:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773075.113196:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534595072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773075.113197:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534594560:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773075.113209:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534594688:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773075.113222:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773075.113228:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773075.113231:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773075.113236:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773075.113241:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2071 00000100:00080000:2.0:1790773075.113247:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773075.113249:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773075/real 1790773075] req@ffff9c4efbcc2a00 x1877759534594688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773075.113258:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773075.113264:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534594688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773075.113268:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773075.113281:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773075.113287:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773075.113289:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534595200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773075.113291:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534594688:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773075.113302:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534594816:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773075.113316:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773075.113320:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1790773075.113325:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773075.113326:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773075.113336:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2071 00000100:00100000:0.0:1790773075.113337:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773075/real 1790773075] req@ffff9c4efbcc1c00 x1877759534594816/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773075.113345:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773075.113351:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534594816/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773075.113356:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773075.113365:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790773075.113368:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773075.113369:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773075.113376:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534594816:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773075.113390:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534595328/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773075.113396:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534594944:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773075.113413:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773075.113417:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773075.113420:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773075.113428:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2071 00000800:00000100:0.0:1790773075.113429:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773075.113435:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773075.113444:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773075.113450:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773075/real 1790773075] req@ffff9c4efbcc1f80 x1877759534594944/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773075.113452:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773075.113467:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534594944/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773075.113474:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534595456/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773075.113478:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773075.113485:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773075.113492:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534594944:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773075.113501:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534595072:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773075.113519:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773075.113527:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773075/real 1790773075] req@ffff9c4efbcc1500 x1877759534595072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773075.113535:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534595072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773075.113546:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773075.113548:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773075.113551:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534595072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773075.113554:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534595200:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773075.113564:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773075.113571:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773075/real 1790773075] req@ffff9c4efbcc1880 x1877759534595200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773075.113578:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534595200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773075.113584:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773075.113586:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773075.113588:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534595200:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773075.113616:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534595328:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773075.113631:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773075.113639:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773075/real 1790773075] req@ffff9c4efbcc2300 x1877759534595328/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773075.113647:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534595328/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773075.113675:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773075.113678:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773075.113682:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534595328:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773075.113687:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534595456:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773075.113699:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773075.113707:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773075/real 1790773075] req@ffff9c4efbcc2680 x1877759534595456/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773075.113715:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534595456/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773075.113721:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773075.113723:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773075.113725:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534595456:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773075.176724:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773075.176724:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773075.176727:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773075.176736:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773075.240713:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773075.240764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773075.240800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773075.368714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773075.368716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773075.432761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773075.496722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773075.560709:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773075.560719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773075.560755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773075.624712:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773075.688743:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773075.688743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773075.688743:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773075.816719:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773075.816720:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773075.816727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773075.816728:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773075.880724:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773075.880755:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773075.880778:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773075.880782:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773075.880789:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773075.880811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773076.008718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773076.072722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773076.136726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773076.200769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773076.200769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773076.200770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773076.200770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773076.264740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773076.264774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773076.264799:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773076.392762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773076.392762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773076.456739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773076.520721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773076.584731:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773076.584737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773076.584742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773076.648723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773076.712715:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773076.712732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773076.712734:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773076.840758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773076.840758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773076.840758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773076.840791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773076.904739:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773076.904739:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773076.904763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773076.904765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773076.904795:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773076.904804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773077.032717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773077.096725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773077.160719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773077.224720:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773077.224721:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773077.224729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773077.224739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773077.288713:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773077.288729:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773077.288790:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773077.416711:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773077.416719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773077.480741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773077.544735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773077.608701:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773077.608703:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773077.608720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773077.672735:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773077.736781:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773077.736781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773077.736781:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773077.864749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773077.864749:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773077.864749:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773077.864749:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773077.928719:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773077.928730:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773077.928733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773077.928733:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773077.928758:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773077.928759:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773078.056719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773078.120747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773078.184745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773078.248782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773078.248783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773078.248791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773078.248791:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773078.312777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773078.312821:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773078.312830:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773078.440721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773078.440749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773078.504727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773078.568734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773078.632708:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773078.632750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773078.632753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773078.696719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773078.760716:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773078.760736:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773078.760779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773078.888765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773078.888765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773078.888765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773078.888772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773078.952728:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773078.952758:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773078.952764:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773078.952779:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773078.952798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773078.952814:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773079.080708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773079.144763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773079.208713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773079.272732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773079.272732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773079.272767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773079.272777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773079.336784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773079.336791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773079.336825:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773079.464719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773079.464764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773079.528720:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773079.592735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773079.656729:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773079.656783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773079.656784:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773079.720736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773079.784736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773079.784736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773079.784766:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773079.912719:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773079.912725:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773079.912725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773079.912775:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773079.976762:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773079.976773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773079.976776:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773079.976777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773079.976814:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773079.976838:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773080.104717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773080.168734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773080.232757:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773080.232760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773080.232770:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773080.232776:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773080.232788:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2076 00000100:00080000:2.0:1790773080.232801:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773080.232814:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773080.232859:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534595584/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773080.232887:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773080.232890:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773080.232892:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773080.232895:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2076 00000100:00080000:2.0:1790773080.232898:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773080.232902:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773080.232912:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773080.232922:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534595712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773080.232935:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534595584:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773080.232938:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773080.232950:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773080.232957:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773080.232964:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2076 00000100:00080000:2.0:1790773080.232969:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773080.232975:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773080.232982:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773080.233001:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534595840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773080.233014:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773080.233020:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773080.233022:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773080.233029:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2076 00000800:00000100:0.0:1790773080.233038:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773080.233039:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773080.233051:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773080.233058:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773080.233073:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534595968/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773080.233082:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773080.233085:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773080.233087:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773080.233091:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2076 00000100:00080000:2.0:1790773080.233094:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773080.233100:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773080.233105:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773080.233119:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773080/real 1790773080] req@ffff9c4efbcc0a80 x1877759534595584/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773080.233120:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534596096/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773080.233150:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773080.233163:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773080.233168:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773080.233180:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2076 00000100:00080000:2.0:1790773080.233186:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773080.233192:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773080.233195:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534595584/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773080.233199:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773080.233212:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534596224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773080.233221:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773080.233223:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773080.233230:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773080.233232:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773080.233236:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773080.233240:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2076 00000100:00080000:2.0:1790773080.233246:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773080.233250:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534595584:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773080.233252:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773080.233259:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773080.233271:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534596352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773080.233279:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773080.233281:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773080.233282:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773080.233288:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2076 00000100:00080000:2.0:1790773080.233290:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773080.233296:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773080.233299:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534595712:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790773080.233300:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773080.233314:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534596480/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773080.233323:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773080.233325:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773080.233328:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773080.233331:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2076 00000100:00080000:2.0:1790773080.233335:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773080.233342:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773080.233346:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790773080.233348:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773080.233364:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534596608/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773080.233378:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773080/real 1790773080] req@ffff9c4efbcc0700 x1877759534595712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773080.233398:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534595712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773080.233417:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773080.233421:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773080.233425:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534595712:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773080.233434:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534595840:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773080.233458:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773080.233471:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773080/real 1790773080] req@ffff9c4efbcc2680 x1877759534595840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773080.233485:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534595840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773080.233518:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773080.233523:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773080.233527:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534595840:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773080.233542:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534595968:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773080.233566:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773080.233579:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773080/real 1790773080] req@ffff9c4efbcc2300 x1877759534595968/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773080.233592:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534595968/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773080.233602:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773080.233607:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773080.233611:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534595968:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773080.233618:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534596096:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773080.233640:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773080.233679:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773080/real 1790773080] req@ffff9c4efbcc1880 x1877759534596096/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773080.233694:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534596096/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773080.233705:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773080.233709:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773080.233713:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534596096:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773080.233721:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534596224:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773080.233742:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773080.233753:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773080/real 1790773080] req@ffff9c4efbcc1500 x1877759534596224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773080.233766:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534596224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773080.233784:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773080.233788:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773080.233792:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534596224:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773080.233799:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534596352:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773080.233819:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773080.233830:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773080/real 1790773080] req@ffff9c4efbcc1f80 x1877759534596352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773080.233843:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534596352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773080.233852:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773080.233856:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773080.233860:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534596352:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773080.233885:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534596480:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773080.233907:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773080.233919:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773080/real 1790773080] req@ffff9c4efbcc1c00 x1877759534596480/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773080.233932:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534596480/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773080.233943:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773080.233947:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773080.233950:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534596480:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773080.233957:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534596608:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773080.233979:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773080.233992:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773080/real 1790773080] req@ffff9c4efbcc2a00 x1877759534596608/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773080.234006:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534596608/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773080.234017:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773080.234021:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773080.234025:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534596608:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773080.296738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773080.296738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773080.296740:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773080.296773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773080.360727:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773080.360733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773080.360892:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773080.488724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773080.488725:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773080.552764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773080.616752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773080.680777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773080.680784:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773080.680784:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773080.744720:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773080.808728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773080.808782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773080.808783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773080.936731:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773080.936735:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773080.936736:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773080.936777:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773081.000751:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773081.000786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773081.000791:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773081.000792:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773081.000798:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773081.000834:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773081.128723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773081.192755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773081.256717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773081.320767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773081.320767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773081.320770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773081.320804:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773081.384775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773081.384823:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773081.384833:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773081.512725:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773081.512763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773081.576730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773081.640778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773081.704805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773081.704809:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773081.704810:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773081.768719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773081.832749:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773081.832749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773081.832771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773081.960730:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773081.960731:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773081.960781:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773081.960808:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773082.024710:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773082.024723:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773082.024736:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773082.024781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773082.024785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773082.024788:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773082.152714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773082.216720:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773082.280736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773082.344731:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773082.344732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773082.344732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773082.344734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773082.408716:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773082.408727:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773082.408802:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773082.536718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773082.536720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773082.600771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773082.664743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773082.728736:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773082.728736:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773082.728737:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773082.792714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773082.856698:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773082.856724:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773082.856732:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773082.984721:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773082.984722:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773082.984728:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773082.984729:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773083.048712:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773083.048726:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773083.048736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773083.048743:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773083.048757:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773083.048762:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773083.176702:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773083.240726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773083.304718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773083.368736:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773083.368736:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773083.368736:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773083.368737:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773083.432710:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773083.432757:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773083.432802:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773083.560701:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773083.560714:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773083.624703:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773083.688705:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773083.752765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773083.752765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773083.752765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773083.816713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773083.880744:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773083.880744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773083.880745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773084.008724:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773084.008738:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773084.008739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773084.009692:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773084.072730:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773084.072738:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773084.072742:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773084.072744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773084.072773:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773084.072776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773084.200722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773084.264729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773084.328726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773084.392726:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773084.392726:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773084.392734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773084.392737:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773084.456743:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773084.456774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773084.456780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773084.584718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773084.584723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773084.648740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773084.712714:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773084.776709:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773084.776712:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773084.776714:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773084.840710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773084.904752:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773084.904752:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773084.904752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773085.032711:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773085.032711:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773085.032714:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773085.032735:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773085.096727:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773085.096731:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773085.096736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773085.096736:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773085.096768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773085.096768:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773085.224722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773085.288731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773085.352719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773085.352719:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773085.352728:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773085.352732:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773085.352740:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2081 00000100:00080000:2.0:1790773085.352749:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773085.352758:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773085.352792:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534596736/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773085.352814:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773085.352818:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773085.352820:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773085.352823:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2081 00000100:00080000:2.0:1790773085.352826:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773085.352829:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773085.352837:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773085.352851:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534596736:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773085.352857:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534596864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773085.352866:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773085.352869:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773085.352870:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773085.352874:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2081 00000100:00080000:2.0:1790773085.352876:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773085.352879:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773085.352883:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773085.352893:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534596992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773085.352900:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773085.352902:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773085.352904:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773085.352906:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2081 00000100:00080000:2.0:1790773085.352909:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773085.352913:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773085.352914:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773085.352920:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773085.352931:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534597120/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773085.352938:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773085.352940:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773085.352942:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773085.352945:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2081 00000100:00080000:2.0:1790773085.352948:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773085.352951:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773085.352956:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773085/real 1790773085] req@ffff9c4efbcc3100 x1877759534596736/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773085.352956:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773085.352968:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534597248/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773085.352975:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534596736/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773085.352978:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773085.352986:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773085.352989:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773085.352996:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2081 00000100:00080000:0.0:1790773085.352998:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773085.352999:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773085.353006:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773085.353007:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773085.353012:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773085.353015:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534596736:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773085.353021:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534597376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773085.353028:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773085.353031:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773085.353032:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773085.353037:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2081 00000100:00100000:0.0:1790773085.353040:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534596864:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773085.353041:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773085.353048:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773085.353052:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773085.353061:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534597504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773085.353066:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773085.353075:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773085.353079:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773085.353080:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773085.353083:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773085/real 1790773085] req@ffff9c4efbcc0380 x1877759534596864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773085.353087:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2081 00000100:00080000:2.0:1790773085.353092:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773085.353098:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773085.353102:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773085.353103:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534596864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773085.353112:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534597632/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773085.353121:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773085.353121:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773085.353128:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773085.353129:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773085.353131:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773085.353136:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534596864:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773085.353137:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2081 00000100:00080000:2.0:1790773085.353145:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773085.353148:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534596992:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773085.353150:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773085.353156:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773085.353166:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534597760/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773085.353173:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773085.353185:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773085/real 1790773085] req@ffff9c4efbcc2a00 x1877759534596992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773085.353197:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534596992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773085.353204:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773085.353207:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773085.353209:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534596992:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773085.353214:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534597120:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773085.353225:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773085.353232:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773085/real 1790773085] req@ffff9c4efbcc1c00 x1877759534597120/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773085.353240:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534597120/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773085.353247:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773085.353249:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773085.353251:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534597120:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773085.353255:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534597248:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773085.353265:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773085.353271:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773085/real 1790773085] req@ffff9c4efbcc1f80 x1877759534597248/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773085.353279:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534597248/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773085.353285:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773085.353287:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773085.353289:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534597248:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773085.353293:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534597376:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773085.353303:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773085.353309:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773085/real 1790773085] req@ffff9c4efbcc1500 x1877759534597376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773085.353318:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534597376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773085.353328:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773085.353330:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773085.353332:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534597376:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773085.353346:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534597504:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773085.353357:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773085.353363:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773085/real 1790773085] req@ffff9c4efbcc1880 x1877759534597504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773085.353371:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534597504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773085.353377:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773085.353379:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773085.353381:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534597504:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773085.353384:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534597632:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773085.353394:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773085.353417:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773085/real 1790773085] req@ffff9c4efbcc2300 x1877759534597632/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773085.353425:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534597632/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773085.353437:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773085.353439:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773085.353442:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534597632:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773085.353446:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534597760:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773085.353458:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773085.353464:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773085/real 1790773085] req@ffff9c4efbcc2680 x1877759534597760/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773085.353472:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534597760/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773085.353479:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773085.353480:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773085.353483:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534597760:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773085.416728:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773085.416729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773085.416731:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773085.416731:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773085.480721:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773085.480744:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773085.480783:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773085.608703:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773085.608714:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773085.672732:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773085.736711:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773085.800726:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773085.800727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773085.800728:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773085.864715:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773085.928715:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773085.928715:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773085.928733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773086.056722:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773086.056722:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773086.056725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773086.056740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773086.120726:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773086.120726:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773086.120726:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773086.120727:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773086.120756:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773086.120760:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773086.248704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773086.312716:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773086.376709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773086.440730:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773086.440735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773086.440764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773086.440764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773086.504732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773086.504737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773086.504794:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773086.632727:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773086.632765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773086.696718:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773086.760721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773086.824708:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773086.824711:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773086.824755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773086.888732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773086.952758:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773086.952759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773086.952768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773087.080719:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773087.080723:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773087.080745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773087.080768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773087.144724:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773087.144750:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773087.144757:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773087.144774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773087.144788:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773087.144793:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773087.272704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773087.336750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773087.400706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773087.464753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773087.464754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773087.464758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773087.464758:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773087.528763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773087.528792:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773087.528823:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773087.656758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773087.656758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773087.720745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773087.784755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773087.848760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773087.848764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773087.848764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773087.912695:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773087.976718:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773087.976727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773087.976727:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773088.104751:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773088.104751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773088.104779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773088.104791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773088.168731:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773088.168731:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773088.168735:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773088.168764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773088.168794:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773088.168799:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773088.296705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773088.360711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773088.424708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773088.488812:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773088.488812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773088.488812:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773088.488813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773088.552725:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773088.552788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773088.552818:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773088.680718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773088.680737:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773088.744763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773088.808726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773088.872744:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773088.872780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773088.872797:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773088.936742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773089.000731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773089.000731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773089.000746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773089.128729:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773089.128798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773089.128798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773089.128798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773089.192722:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773089.192735:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773089.192752:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773089.192766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773089.192779:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773089.192795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773089.320713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773089.384716:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773089.448716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773089.512709:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773089.512718:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773089.512723:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773089.512754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773089.576710:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773089.576781:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773089.576804:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773089.704725:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773089.704739:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773089.768747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773089.832717:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773089.896753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773089.896781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773089.896798:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773089.960804:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773090.024718:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773090.024733:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773090.024788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773090.152728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773090.152729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773090.152731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773090.152739:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773090.216714:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773090.216725:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773090.216733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773090.216742:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773090.216757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773090.216761:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773090.344709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773090.408723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773090.472731:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773090.472742:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1790773090.472744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773090.472747:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773090.472760:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2086 00000100:00080000:2.0:1790773090.472774:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773090.472788:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773090.472837:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534597888/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773090.472866:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773090.472870:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773090.472871:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773090.472875:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2086 00000100:00080000:2.0:1790773090.472878:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773090.472889:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773090.472900:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773090.472916:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534597888:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773090.472917:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534598016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773090.472934:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773090.472937:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773090.472939:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773090.472942:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2086 00000100:00080000:2.0:1790773090.472945:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773090.472950:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773090.472955:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773090.472966:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534598144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773090.472975:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773090.472979:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773090.472981:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773090.472984:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2086 00000100:00080000:2.0:1790773090.472987:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773090.472991:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773090.472995:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773090.473005:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534598272/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773090.473012:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773090.473017:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773090.473018:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773090.473022:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773090.473028:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2086 00000100:00080000:2.0:1790773090.473031:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773090.473035:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773090.473039:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773090.473048:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534598400/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773090.473055:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773090.473059:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773090.473060:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773090.473064:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2086 00000100:00080000:2.0:1790773090.473067:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773090.473071:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773090.473075:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773090.473083:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773090/real 1790773090] req@ffff9c4efbcc0700 x1877759534597888/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773090.473086:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534598528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773090.473123:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773090.473127:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773090.473129:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773090.473134:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2086 00000100:00080000:2.0:1790773090.473139:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773090.473140:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534597888/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773090.473145:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773090.473151:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773090.473162:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534598656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773090.473166:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773090.473173:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773090.473176:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773090.473179:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773090.473180:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773090.473190:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2086 00000100:00100000:0.0:1790773090.473191:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534597888:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773090.473193:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773090.473200:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773090.473205:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773090.473213:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534598784/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773090.473220:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773090.473223:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773090.473225:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773090.473249:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2086 00000100:00100000:0.0:1790773090.473251:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534598016:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773090.473253:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773090.473262:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773090.473270:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773090.473285:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534598912/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773090.473336:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773090.473357:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773090/real 1790773090] req@ffff9c4efbcc0a80 x1877759534598016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773090.473373:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534598016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773090.473393:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773090.473398:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773090.473403:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534598016:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773090.473412:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534598144:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773090.473437:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773090.473450:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773090/real 1790773090] req@ffff9c4efbcc2680 x1877759534598144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773090.473464:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534598144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773090.473474:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773090.473479:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773090.473483:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534598144:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773090.473491:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534598272:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773090.473516:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773090.473530:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773090/real 1790773090] req@ffff9c4efbcc2300 x1877759534598272/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773090.473546:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534598272/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773090.473558:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773090.473563:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773090.473567:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534598272:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773090.473575:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534598400:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773090.473597:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773090.473610:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773090/real 1790773090] req@ffff9c4efbcc1880 x1877759534598400/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773090.473626:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534598400/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773090.473639:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773090.473644:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773090.473672:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534598400:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773090.473681:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534598528:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773090.473707:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773090.473720:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773090/real 1790773090] req@ffff9c4efbcc1500 x1877759534598528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773090.473741:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534598528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773090.473759:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773090.473763:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773090.473767:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534598528:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773090.473774:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534598656:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773090.473797:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773090.473811:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773090/real 1790773090] req@ffff9c4efbcc1f80 x1877759534598656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773090.473825:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534598656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773090.473836:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773090.473841:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773090.473845:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534598656:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773090.473853:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534598784:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773090.473877:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773090.473890:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773090/real 1790773090] req@ffff9c4efbcc1c00 x1877759534598784/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773090.473904:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534598784/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773090.473923:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773090.473927:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773090.473932:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534598784:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773090.473961:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534598912:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773090.473984:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773090.473997:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773090/real 1790773090] req@ffff9c4efbcc2a00 x1877759534598912/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773090.474010:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534598912/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773090.474021:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773090.474025:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773090.474030:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534598912:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773090.536725:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773090.536725:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773090.536729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773090.536730:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773090.600808:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773090.600854:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773090.600901:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773090.728710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773090.728715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773090.792769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773090.856732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773090.920747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773090.920747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773090.920748:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773090.984719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773091.048728:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773091.048728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773091.048729:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773091.176715:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773091.176754:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773091.176776:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773091.176782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773091.240737:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773091.240737:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773091.240744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773091.240786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773091.240796:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773091.240812:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773091.368709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773091.432761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773091.496722:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773091.560721:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773091.560749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773091.560749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773091.560750:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773091.624779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773091.624795:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773091.624805:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773091.752759:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773091.752762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773091.816714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773091.880715:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773091.944702:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773091.944720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773091.944761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773092.008707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773092.072747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773092.072748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773092.072772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773092.200737:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773092.200737:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773092.200738:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773092.200769:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773092.264817:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773092.264835:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773092.264836:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773092.264837:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773092.264844:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773092.264874:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773092.392706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773092.456737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773092.520708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773092.584735:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773092.584741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773092.584742:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773092.584771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773092.648770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773092.648828:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773092.648837:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773092.776714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773092.776735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773092.840729:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773092.904721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773092.968709:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773092.968717:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773092.968759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773093.032781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773093.096761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773093.096761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773093.096782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773093.224725:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773093.224756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773093.224768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773093.224775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773093.288728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773093.288757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773093.288789:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773093.288790:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773093.288800:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773093.288805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773093.416712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773093.480707:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773093.544718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773093.608700:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773093.608744:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773093.608787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773093.608791:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773093.672717:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773093.672780:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773093.672801:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773093.800782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773093.800785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773093.864736:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773093.928745:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773093.992715:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773093.992715:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773093.992781:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773094.056736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773094.120718:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773094.120733:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773094.120750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773094.248752:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773094.248770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773094.248786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773094.248790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773094.312741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773094.312751:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773094.312796:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773094.312807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773094.312832:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773094.312836:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773094.440715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773094.504707:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773094.568712:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773094.632723:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773094.632723:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773094.632730:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773094.632748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773094.696794:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773094.696861:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773094.696864:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773094.824714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773094.824738:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773094.888716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773094.952753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773095.016707:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773095.016715:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773095.016752:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773095.080719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773095.144752:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773095.144752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773095.144761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773095.272731:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773095.272732:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773095.272734:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773095.272742:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773095.336750:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773095.336763:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773095.336789:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773095.336800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773095.336810:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773095.336832:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773095.464708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773095.528703:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773095.592734:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773095.592735:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773095.592747:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773095.592754:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773095.592771:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2091 00000100:00080000:2.0:1790773095.592786:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773095.592802:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773095.592862:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534599040/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773095.592895:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773095.592898:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773095.592900:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773095.592904:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2091 00000100:00080000:2.0:1790773095.592909:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773095.592916:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773095.592927:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773095.592943:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534599168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773095.592944:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534599040:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773095.592972:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773095.592975:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773095.592978:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773095.592984:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2091 00000100:00080000:2.0:1790773095.592988:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773095.592992:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773095.592998:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773095.593009:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534599296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773095.593017:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773095.593021:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773095.593022:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773095.593026:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2091 00000800:00000100:0.0:1790773095.593032:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773095.593032:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773095.593045:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773095.593052:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773095.593063:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534599424/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773095.593071:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773095.593074:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773095.593076:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773095.593080:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2091 00000100:00080000:2.0:1790773095.593084:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773095.593088:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773095/real 1790773095] req@ffff9c4efbcc0380 x1877759534599040/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773095.593092:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773095.593104:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773095.593123:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534599552/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773095.593134:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773095.593136:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534599040/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773095.593146:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773095.593151:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773095.593151:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773095.593164:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773095.593165:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2091 00000100:00080000:2.0:1790773095.593173:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773095.593179:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534599040:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773095.593185:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773095.593193:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773095.593227:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534599168:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773095.593249:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773095.593259:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773095/real 1790773095] req@ffff9c4efbcc3100 x1877759534599168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773095.593268:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534599168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773095.593282:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773095.593285:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773095.593288:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534599168:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773095.593293:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534599296:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773095.593305:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773095.593312:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773095/real 1790773095] req@ffff9c4efbcc2a00 x1877759534599296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773095.593320:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534599296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773095.593330:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773095.593333:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773095.593335:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534599296:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773095.593339:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534599424:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773095.593354:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534599680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773095.593355:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773095.593367:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773095/real 1790773095] req@ffff9c4efbcc1c00 x1877759534599424/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773095.593411:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534599424/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773095.593415:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773095.593428:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773095.593429:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773095.593431:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773095.593437:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773095.593444:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2091 00000100:00100000:0.0:1790773095.593446:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534599424:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773095.593452:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773095.593457:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534599552:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773095.593462:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773095.593472:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773095.593486:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773095.593486:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534599808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773095.593500:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773095.593504:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773095/real 1790773095] req@ffff9c4efbcc1f80 x1877759534599552/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773095.593509:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773095.593516:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773095.593534:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2091 00000100:00100000:0.0:1790773095.593541:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534599552/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773095.593543:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773095.593559:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773095.593574:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773095.593575:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773095.593582:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773095.593588:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534599936/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773095.593593:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534599552:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773095.593616:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773095.593619:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773095.593623:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773095.593628:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2091 00000100:00080000:2.0:1790773095.593633:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773095.593640:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773095.593643:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773095.593645:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534599680:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773095.593683:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534600064/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773095.593713:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773095.593731:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773095/real 1790773095] req@ffff9c4efbcc1500 x1877759534599680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773095.593745:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534599680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773095.593762:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773095.593767:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773095.593770:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534599680:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773095.593778:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534599808:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773095.593801:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773095.593813:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773095/real 1790773095] req@ffff9c4efbcc1880 x1877759534599808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773095.593825:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534599808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773095.593833:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773095.593837:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773095.593841:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534599808:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773095.593848:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534599936:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773095.593866:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773095.593878:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773095/real 1790773095] req@ffff9c4efbcc2300 x1877759534599936/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773095.593892:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534599936/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773095.593903:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773095.593907:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773095.593910:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534599936:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773095.593928:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534600064:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773095.593949:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773095.593961:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773095/real 1790773095] req@ffff9c4efbcc2680 x1877759534600064/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773095.593975:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534600064/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773095.593986:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773095.593989:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773095.593993:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534600064:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773095.656748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773095.656771:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773095.656789:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773095.656798:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773095.720794:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773095.720835:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773095.720867:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773095.848769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773095.848771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773095.912714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773095.976755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773096.040698:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773096.040702:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773096.040761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773096.104718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773096.168715:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773096.168720:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773096.168741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773096.296810:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773096.296812:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773096.296828:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773096.296828:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773096.360738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773096.360738:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773096.360763:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773096.360786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773096.360797:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773096.360797:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773096.488725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773096.552767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773096.616703:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773096.680716:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773096.680783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773096.680796:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773096.680797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773096.744727:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773096.744780:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773096.744789:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773096.872714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773096.872810:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773096.936708:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773097.000718:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773097.064709:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773097.064735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773097.064735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773097.128714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773097.192731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773097.192734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773097.192737:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773097.320738:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773097.320749:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773097.320757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773097.320768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773097.384731:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773097.384748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773097.384760:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773097.384764:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773097.384796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773097.384801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773097.512704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773097.576711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773097.640707:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773097.704744:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773097.704754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773097.704777:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773097.704828:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773097.768717:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773097.768748:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773097.768806:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773097.896718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773097.896728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773097.960705:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773098.024759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773098.088730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773098.088730:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773098.088751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773098.152740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773098.216717:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773098.216746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773098.216764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773098.344772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773098.344784:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773098.344803:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773098.344805:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773098.408746:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773098.408746:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773098.408770:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773098.408793:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773098.408798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773098.408806:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773098.536714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773098.600758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773098.664709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773098.728742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773098.728743:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773098.728755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773098.728780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773098.792776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773098.792788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773098.792821:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773098.920708:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773098.920736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773098.984713:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773099.048741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773099.112758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773099.112758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773099.112759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773099.176708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773099.240719:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773099.240720:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773099.240750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773099.368725:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773099.368725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773099.368740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773099.368761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773099.432748:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773099.432748:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773099.432749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773099.432790:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773099.432794:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773099.432802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773099.560710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773099.624772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773099.688707:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773099.752759:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773099.752792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773099.752805:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773099.752810:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773099.816724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773099.816762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773099.816796:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773099.944732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773099.944744:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773100.008735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773100.072722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773100.136714:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773100.136717:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773100.136724:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773100.200724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773100.264708:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773100.264711:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773100.264729:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773100.392770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773100.392770:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773100.392771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773100.392771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773100.456734:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773100.456735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773100.456735:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773100.456739:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773100.456764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773100.456770:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773100.584731:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773100.648747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773100.712721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773100.712725:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773100.712732:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773100.712735:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773100.712743:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2096 00000100:00080000:2.0:1790773100.712752:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773100.712762:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773100.712796:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534600192/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773100.712819:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773100.712822:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773100.712824:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773100.712828:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2096 00000100:00080000:2.0:1790773100.712831:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773100.712835:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773100.712843:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773100.712852:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534600192:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773100.712857:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534600320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773100.712866:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773100.712869:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773100.712871:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773100.712874:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2096 00000100:00080000:2.0:1790773100.712877:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773100.712880:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773100.712883:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773100.712892:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534600448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773100.712899:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773100.712902:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773100.712903:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773100.712906:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2096 00000100:00080000:2.0:1790773100.712909:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773100.712912:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773100.712916:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773100.712921:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773100.712930:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534600576/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773100.712937:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773100.712940:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773100.712941:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773100.712946:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2096 00000100:00080000:2.0:1790773100.712949:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773100.712953:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773100.712956:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773100/real 1790773100] req@ffff9c4efbcc0a80 x1877759534600192/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773100.712958:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773100.712971:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534600704/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773100.712978:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773100.712979:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534600192/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773100.712984:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773100.712988:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773100.712997:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2096 00000100:00080000:0.0:1790773100.712999:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773100.713000:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773100.713007:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773100.713008:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773100.713013:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773100.713015:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534600192:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773100.713024:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534600832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773100.713030:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773100.713033:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773100.713034:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773100.713038:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534600320:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773100.713040:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2096 00000100:00080000:2.0:1790773100.713046:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773100.713050:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773100.713055:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773100.713063:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773100.713065:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534600960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773100.713076:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773100/real 1790773100] req@ffff9c4efbcc0700 x1877759534600320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773100.713076:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773100.713081:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773100.713086:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773100.713096:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2096 00000100:00100000:0.0:1790773100.713111:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534600320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773100.713117:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773100.713128:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773100.713130:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790773100.713134:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773100.713136:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773100.713141:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534600320:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773100.713147:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534600448:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773100.713152:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534601088/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773100.713161:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773100.713165:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1790773100.713167:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773100.713168:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773100.713174:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2096 00000100:00080000:2.0:1790773100.713179:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773100.713181:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773100/real 1790773100] req@ffff9c4efbcc2680 x1877759534600448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773100.713188:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773100.713196:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534600448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773100.713197:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790773100.713209:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773100.713213:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773100.713218:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534601216/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773100.713219:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534600448:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773100.713230:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534600576:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773100.713258:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773100.713282:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773100/real 1790773100] req@ffff9c4efbcc2300 x1877759534600576/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773100.713296:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534600576/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773100.713303:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773100.713306:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773100.713308:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534600576:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773100.713313:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534600704:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773100.713328:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773100.713336:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773100/real 1790773100] req@ffff9c4efbcc1880 x1877759534600704/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773100.713343:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534600704/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773100.713350:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773100.713352:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773100.713354:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534600704:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773100.713358:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534600832:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773100.713368:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773100.713374:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773100/real 1790773100] req@ffff9c4efbcc1500 x1877759534600832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773100.713382:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534600832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773100.713393:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773100.713395:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773100.713397:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534600832:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773100.713412:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534600960:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773100.713423:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773100.713430:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773100/real 1790773100] req@ffff9c4efbcc1f80 x1877759534600960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773100.713438:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534600960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773100.713443:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773100.713446:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773100.713448:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534600960:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773100.713452:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534601088:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773100.713461:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773100.713466:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773100/real 1790773100] req@ffff9c4efbcc1c00 x1877759534601088/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773100.713474:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534601088/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773100.713485:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773100.713488:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773100.713490:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534601088:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773100.713494:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534601216:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773100.713503:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773100.713509:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773100/real 1790773100] req@ffff9c4efbcc2a00 x1877759534601216/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773100.713517:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534601216/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773100.713523:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773100.713525:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773100.713527:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534601216:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773100.776717:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773100.776717:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773100.776722:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773100.776723:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773100.840716:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773100.840762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773100.840770:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773100.968712:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773100.968723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773101.032705:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773101.096732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773101.160746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773101.160746:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773101.160746:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773101.224777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773101.288715:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773101.288723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773101.288724:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773101.416713:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773101.416730:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773101.416731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773101.416733:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773101.480701:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773101.480723:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773101.480727:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773101.480741:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773101.480744:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773101.480754:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773101.608724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773101.672702:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773101.736732:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773101.800734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773101.800735:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773101.800735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773101.800742:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773101.864724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773101.864736:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773101.864804:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773101.992730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773101.992731:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773102.056751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773102.120746:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773102.184755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773102.184755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773102.184788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773102.248729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773102.312712:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773102.312751:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773102.312759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773102.440734:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773102.440734:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773102.440760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773102.440768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773102.504736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773102.504736:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773102.504745:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773102.504770:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773102.504795:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773102.504796:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773102.632708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773102.696724:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773102.760710:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773102.824745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773102.824745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773102.824745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773102.824766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773102.888726:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773102.888750:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773102.888921:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773103.016716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773103.016730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773103.080766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773103.144767:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773103.208762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773103.208762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773103.208763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773103.272720:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773103.336777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773103.336781:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773103.336781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773103.464746:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773103.464746:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773103.464747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773103.464747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773103.528718:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773103.528724:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773103.528737:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773103.528752:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773103.528767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773103.528782:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773103.656712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773103.720725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773103.784697:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773103.848762:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773103.848762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773103.848762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773103.848779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773103.912732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773103.912754:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773103.912899:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773104.040732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773104.040732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773104.104780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773104.168780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773104.232714:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773104.232743:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773104.232763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773104.296710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773104.360759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773104.360769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773104.360769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773104.488744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773104.488745:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773104.488751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773104.488755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773104.552774:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773104.552779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773104.552787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773104.552805:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773104.552815:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773104.552820:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773104.680704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773104.744770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773104.808725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773104.872735:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773104.872746:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773104.872761:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773104.872761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773104.936734:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773104.936737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773104.936883:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773105.064719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773105.064775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773105.128741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773105.192736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773105.256739:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773105.256740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773105.256740:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773105.320721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773105.384725:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773105.384725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773105.384727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773105.512717:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773105.512717:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773105.512722:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773105.512775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773105.576743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773105.576743:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773105.576754:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773105.576798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773105.576821:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773105.576822:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773105.704753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773105.768722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773105.832755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773105.832756:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773105.832772:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773105.832778:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773105.832791:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2101 00000100:00080000:2.0:1790773105.832802:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773105.832815:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773105.832858:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534601344/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773105.832889:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773105.832893:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773105.832894:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773105.832898:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2101 00000100:00080000:2.0:1790773105.832901:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773105.832906:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773105.832915:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773105.832929:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534601472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773105.832937:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773105.832941:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773105.832945:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773105.832948:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534601344:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773105.832950:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2101 00000100:00080000:2.0:1790773105.832958:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773105.832978:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773105.832982:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773105.832996:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534601600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773105.833017:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773105.833020:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773105.833022:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773105.833026:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2101 00000100:00080000:2.0:1790773105.833029:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773105.833034:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773105.833038:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773105.833047:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534601728/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773105.833053:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773105.833056:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773105.833058:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773105.833060:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2101 00000100:00080000:2.0:1790773105.833064:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773105.833074:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773105.833074:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773105.833085:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773105.833094:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534601856/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773105.833102:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773105.833105:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773105.833107:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773105.833112:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2101 00000100:00080000:2.0:1790773105.833115:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773105.833118:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773105.833123:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773105.833133:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534601984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773105.833140:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773105.833145:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1790773105.833146:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773105/real 1790773105] req@ffff9c4efbcc3100 x1877759534601344/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773105.833147:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773105.833156:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2101 00000100:00080000:2.0:1790773105.833161:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773105.833168:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773105.833174:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773105.833190:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534602112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773105.833203:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534601344/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773105.833206:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773105.833218:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773105.833222:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773105.833237:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2101 00000100:00080000:2.0:1790773105.833243:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773105.833247:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773105.833253:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790773105.833253:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773105.833269:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773105.833270:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534602240/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773105.833292:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534601344:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773105.833294:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773105.833306:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773105.833309:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773105.833314:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2101 00000100:00080000:2.0:1790773105.833316:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773105.833319:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773105.833322:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773105.833330:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534602368/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773105.833340:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534601472:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773105.833376:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773105.833393:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773105/real 1790773105] req@ffff9c4efbcc0380 x1877759534601472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773105.833407:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534601472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773105.833428:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773105.833432:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773105.833436:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534601472:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773105.833444:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534601600:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773105.833467:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773105.833480:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773105/real 1790773105] req@ffff9c4efbcc2a00 x1877759534601600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773105.833501:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534601600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773105.833512:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773105.833516:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773105.833521:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534601600:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773105.833528:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534601728:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773105.833551:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773105.833564:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773105/real 1790773105] req@ffff9c4efbcc1c00 x1877759534601728/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773105.833577:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534601728/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773105.833588:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773105.833592:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773105.833596:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534601728:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773105.833602:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534601856:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773105.833623:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773105.833633:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773105/real 1790773105] req@ffff9c4efbcc1f80 x1877759534601856/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773105.833645:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534601856/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773105.833687:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773105.833691:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773105.833695:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534601856:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773105.833703:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534601984:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773105.833724:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773105.833736:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773105/real 1790773105] req@ffff9c4efbcc1500 x1877759534601984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773105.833749:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534601984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773105.833766:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773105.833770:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773105.833774:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534601984:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773105.833782:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534602112:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773105.833801:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773105.833812:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773105/real 1790773105] req@ffff9c4efbcc1880 x1877759534602112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773105.833825:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534602112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773105.833834:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773105.833838:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773105.833843:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534602112:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773105.833850:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534602240:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773105.833870:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773105.833881:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773105/real 1790773105] req@ffff9c4efbcc2300 x1877759534602240/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773105.833893:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534602240/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773105.833904:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773105.833908:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773105.833912:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534602240:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773105.833939:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534602368:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773105.833958:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773105.833969:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773105/real 1790773105] req@ffff9c4efbcc2680 x1877759534602368/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773105.833982:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534602368/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773105.833992:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773105.833996:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773105.834014:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534602368:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773105.896762:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773105.896762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773105.896763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773105.896770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773105.960732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773105.960752:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773105.960893:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773106.088732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773106.088763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773106.152760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773106.216758:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773106.280740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773106.280776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773106.280786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773106.344740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773106.408809:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773106.408809:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773106.408811:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773106.536735:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773106.536739:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773106.536739:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773106.536761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773106.600751:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773106.600751:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773106.600775:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773106.600785:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773106.600817:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773106.600829:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773106.728778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773106.792764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773106.856755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773106.920774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773106.920807:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773106.920818:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773106.920822:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773106.984771:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773106.984808:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773106.985045:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773107.112775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773107.112794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773107.176825:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773107.240766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773107.304775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773107.304844:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773107.304858:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773107.368856:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773107.432760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773107.432781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773107.432802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773107.560780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773107.560787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773107.560863:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773107.560863:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773107.624763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773107.624779:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773107.624812:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773107.624814:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773107.624852:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773107.624873:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773107.752776:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773107.816721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773107.880716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773107.944728:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773107.944729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773107.944746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773107.944747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773108.008812:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773108.008895:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773108.009030:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773108.136756:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773108.136781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773108.200804:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773108.264799:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773108.328756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773108.328768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773108.328799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773108.392708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773108.456819:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773108.456833:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773108.456834:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773108.584739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773108.584765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773108.584772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773108.584773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773108.648743:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773108.648743:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773108.648757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773108.648779:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773108.648807:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773108.648811:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773108.776720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773108.840746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773108.904721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773108.968719:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773108.968731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773108.968765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773108.968791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773109.032738:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773109.032842:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773109.032911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773109.160762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773109.160785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773109.224809:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773109.288819:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773109.352769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773109.352813:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773109.352814:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773109.416765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773109.480760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773109.480776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773109.480797:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773109.608762:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773109.608787:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773109.608800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773109.608840:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773109.672842:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773109.672863:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773109.672883:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773109.672893:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773109.672977:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773109.672986:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773109.800777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773109.864759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773109.928747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773109.992780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773109.992836:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773109.992851:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773109.992862:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773110.056784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773110.056890:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773110.056901:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773110.184796:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773110.184802:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773110.248770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773110.312766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773110.376771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773110.376792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773110.376796:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773110.440788:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773110.504772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773110.504812:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773110.504816:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773110.632769:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773110.632783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773110.632807:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773110.632823:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773110.696808:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773110.696816:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773110.696868:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773110.696868:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773110.696935:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773110.696958:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773110.824779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773110.888779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773110.952834:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773110.952839:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773110.952862:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773110.952871:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773110.952888:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2106 00000100:00080000:2.0:1790773110.952932:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773110.952955:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773110.953027:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534602496/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773110.953088:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773110.953097:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773110.953101:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773110.953111:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2106 00000100:00080000:2.0:1790773110.953120:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773110.953132:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773110.953154:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773110.953182:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534602496:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773110.953190:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534602624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773110.953214:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773110.953222:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773110.953225:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773110.953233:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2106 00000100:00080000:2.0:1790773110.953242:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773110.953255:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773110.953264:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773110.953284:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534602752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773110.953291:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773110.953314:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773110.953322:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773110.953325:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773110.953336:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2106 00000100:00080000:2.0:1790773110.953349:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773110.953371:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773110.953372:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773110/real 1790773110] req@ffff9c4efbcc0700 x1877759534602496/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773110.953401:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773110.953411:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534602496/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773110.953430:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773110.953445:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773110.953445:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534602880/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773110.953468:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534602496:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773110.953504:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773110.953515:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773110.953520:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773110.953526:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534602624:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773110.953541:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2106 00000100:00080000:2.0:1790773110.953556:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773110.953569:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773110.953575:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773110.953593:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773110/real 1790773110] req@ffff9c4efbcc0a80 x1877759534602624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773110.953595:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773110.953630:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534602624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773110.953641:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534603008/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773110.953719:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773110.953738:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773110.953738:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773110.953759:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1790773110.953760:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534602624:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773110.953771:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773110.953783:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2106 00000100:00100000:0.0:1790773110.953784:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534602752:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773110.953798:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773110.953815:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773110.953824:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773110.953834:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773110.953853:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534603136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773110.953865:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773110/real 1790773110] req@ffff9c4efbcc2680 x1877759534602752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773110.953986:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773110.954005:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773110.954015:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773110.954035:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2106 00000100:00080000:2.0:1790773110.954044:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773110.954054:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534602752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773110.954055:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773110.954075:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773110.954087:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773110.954101:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773110.954114:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534603264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773110.954122:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534602752:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773110.954162:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534602880:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773110.954165:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773110.954187:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773110.954192:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773110.954203:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2106 00000100:00080000:2.0:1790773110.954211:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773110.954221:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773110.954230:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790773110.954231:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773110.954271:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534603392/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773110.954273:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773110/real 1790773110] req@ffff9c4efbcc2300 x1877759534602880/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773110.954374:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534602880/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773110.954381:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773110.954401:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773110.954402:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773110.954422:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773110.954423:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773110.954441:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534602880:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773110.954450:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2106 00000100:00080000:2.0:1790773110.954462:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773110.954475:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773110.954485:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773110.954488:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534603008:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773110.954534:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773110.954534:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534603520/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773110.954560:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773110/real 1790773110] req@ffff9c4efbcc1880 x1877759534603008/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773110.954615:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534603008/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773110.954645:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773110.954650:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773110.954690:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534603008:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773110.954698:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534603136:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773110.954723:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773110.954738:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773110/real 1790773110] req@ffff9c4efbcc1500 x1877759534603136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773110.954753:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534603136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773110.954773:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773110.954778:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773110.954783:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534603136:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773110.954791:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534603264:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773110.954811:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773110.954824:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773110/real 1790773110] req@ffff9c4efbcc1f80 x1877759534603264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773110.954839:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534603264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773110.954849:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773110.954853:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773110.954858:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534603264:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773110.954866:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534603392:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773110.954891:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773110.954943:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773110/real 1790773110] req@ffff9c4efbcc1c00 x1877759534603392/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773110.954968:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534603392/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773110.954990:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773110.954998:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773110.955006:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534603392:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773110.955037:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534603520:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773110.955069:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773110.955092:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773110/real 1790773110] req@ffff9c4efbcc2a00 x1877759534603520/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773110.955116:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534603520/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773110.955136:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773110.955144:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773110.955152:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534603520:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773111.016811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773111.016828:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773111.016829:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773111.016838:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773111.080857:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773111.080951:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773111.080999:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773111.208759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773111.208791:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773111.272755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773111.336783:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773111.400808:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773111.400814:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773111.400819:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773111.464746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773111.528818:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773111.528854:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773111.528858:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773111.656803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773111.656820:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773111.656858:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773111.656870:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773111.720796:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773111.720836:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773111.720919:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773111.720924:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773111.720926:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773111.720995:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773111.848777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773111.912734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773111.976766:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773112.040759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773112.040772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773112.040800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773112.040806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773112.104776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773112.104830:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773112.104996:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773112.232791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773112.232811:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773112.296786:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773112.360795:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773112.424756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773112.424763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773112.424802:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773112.488742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773112.552766:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773112.552804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773112.552811:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773112.680760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773112.680806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773112.680821:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773112.680830:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773112.744780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773112.744800:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773112.744826:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773112.744828:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773112.744888:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773112.744915:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773112.872773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773112.936763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773113.000734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773113.064817:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773113.064817:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773113.064824:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773113.064826:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773113.128783:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773113.128947:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773113.129023:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773113.256749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773113.256793:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773113.320775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773113.384800:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773113.448763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773113.448776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773113.448801:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773113.512768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773113.576762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773113.576802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773113.576811:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773113.704775:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773113.704780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773113.704824:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773113.704840:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773113.768770:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773113.768803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773113.768827:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773113.768841:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773113.768876:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773113.768920:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773113.896753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773113.960768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773114.024751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773114.088787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773114.088796:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773114.088796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773114.088797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773114.152803:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773114.152933:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773114.152938:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773114.280772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773114.280800:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773114.344762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773114.408782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773114.472779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773114.472779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773114.472802:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773114.536765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773114.600760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773114.600766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773114.600810:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773114.728856:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773114.728886:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773114.728887:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773114.728889:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773114.792786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773114.792797:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773114.792806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773114.792849:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773114.792873:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773114.792881:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773114.920769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773114.984743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773115.048735:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773115.112758:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773115.112789:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773115.112795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773115.112805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773115.176845:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773115.176889:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773115.176998:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773115.304760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773115.304804:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773115.368774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773115.432760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773115.496757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773115.496781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773115.496783:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773115.560743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773115.624758:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773115.624777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773115.624783:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773115.752760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773115.752772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773115.752783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773115.752794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773115.816746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773115.816798:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773115.816843:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773115.816843:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773115.816861:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773115.816910:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773115.944750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773116.008749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773116.072761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773116.072772:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773116.072783:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773116.072789:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773116.072826:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2111 00000100:00080000:2.0:1790773116.072841:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773116.072856:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773116.072905:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534603648/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773116.072953:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773116.072959:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773116.072962:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773116.072969:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2111 00000100:00080000:2.0:1790773116.072974:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773116.072982:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773116.072997:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773116.073008:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534603648:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773116.073020:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534603776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773116.073039:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773116.073047:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773116.073052:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773116.073061:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2111 00000100:00080000:2.0:1790773116.073071:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773116.073081:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773116.073093:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773116.073103:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773116.073130:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534603904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773116.073156:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773116.073163:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773116/real 1790773116] req@ffff9c4efbcc0380 x1877759534603648/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773116.073166:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773116.073172:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773116.073188:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2111 00000100:00080000:2.0:1790773116.073202:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773116.073210:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534603648/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773116.073215:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773116.073226:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773116.073246:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534604032/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773116.073250:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773116.073262:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773116.073269:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534603648:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773116.073295:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773116.073300:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773116.073304:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773116.073313:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2111 00000100:00080000:2.0:1790773116.073319:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773116.073322:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534603776:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773116.073331:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773116.073348:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773116.073369:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534604160/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773116.073384:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773116.073390:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773116.073393:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773116.073404:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773116.073408:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2111 00000100:00080000:2.0:1790773116.073418:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773116.073435:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773116.073443:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773116.073446:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773116/real 1790773116] req@ffff9c4efbcc3100 x1877759534603776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773116.073464:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534604288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773116.073474:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534603776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773116.073500:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773116.073510:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773116.073513:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773116.073523:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2111 00000100:00080000:0.0:1790773116.073526:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773116.073529:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773116.073543:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773116.073545:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773116.073554:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773116.073557:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534603776:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773116.073582:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534603904:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773116.073582:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534604416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773116.073610:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773116.073615:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773116.073620:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773116.073628:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2111 00000800:00000100:0.0:1790773116.073636:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773116.073637:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773116.073688:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773116.073696:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773116.073717:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773116/real 1790773116] req@ffff9c4efbcc2a00 x1877759534603904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773116.073719:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534604544/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773116.073777:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773116.073785:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773116.073790:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773116.073794:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534603904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773116.073829:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2111 00000100:00080000:2.0:1790773116.073842:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773116.073858:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773116.073860:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790773116.073867:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790773116.073877:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773116.073890:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534603904:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773116.073895:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534604672/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773116.073920:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534604032:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773116.073965:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773116.073982:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773116/real 1790773116] req@ffff9c4efbcc1c00 x1877759534604032/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773116.073999:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534604032/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773116.074012:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773116.074017:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773116.074022:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534604032:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773116.074047:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534604160:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773116.074068:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773116.074080:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773116/real 1790773116] req@ffff9c4efbcc1f80 x1877759534604160/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773116.074095:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534604160/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773116.074108:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773116.074112:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773116.074117:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534604160:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773116.074124:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534604288:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773116.074142:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773116.074154:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773116/real 1790773116] req@ffff9c4efbcc1500 x1877759534604288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773116.074169:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534604288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773116.074189:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773116.074193:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773116.074197:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534604288:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773116.074205:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534604416:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773116.074224:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773116.074236:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773116/real 1790773116] req@ffff9c4efbcc1880 x1877759534604416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773116.074251:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534604416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773116.074261:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773116.074266:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773116.074271:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534604416:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773116.074279:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534604544:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773116.074298:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773116.074311:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773116/real 1790773116] req@ffff9c4efbcc2300 x1877759534604544/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773116.074335:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534604544/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773116.074356:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773116.074364:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773116.074372:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534604544:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773116.074385:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534604672:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773116.074425:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773116.074444:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773116/real 1790773116] req@ffff9c4efbcc2680 x1877759534604672/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773116.074460:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534604672/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773116.074473:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773116.074477:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773116.074482:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534604672:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773116.136768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773116.136782:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773116.136824:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773116.136895:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773116.200786:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773116.200880:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773116.201025:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773116.328774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773116.328784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773116.392774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773116.456772:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773116.520768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773116.520856:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773116.520859:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773116.584751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773116.648760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773116.648786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773116.648815:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773116.776806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773116.776812:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773116.776813:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773116.776851:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773116.840805:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773116.840835:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773116.840862:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773116.840890:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773116.840904:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773116.840917:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773116.968745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773117.032760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773117.096713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773117.160721:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773117.160722:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773117.160725:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773117.160753:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773117.224758:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773117.224814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773117.224833:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773117.352763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773117.352828:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773117.416730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773117.480794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773117.544755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773117.544756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773117.544814:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773117.608693:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773117.672752:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773117.672824:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773117.672830:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773117.800716:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773117.800725:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773117.800726:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773117.800833:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773117.864818:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773117.864820:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773117.864848:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773117.864878:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773117.864881:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773117.864907:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773117.992836:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773118.056758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773118.120835:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773118.184705:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773118.184806:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773118.184812:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773118.184827:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773118.248802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773118.248904:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773118.248990:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773118.376816:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773118.376818:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773118.440819:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773118.504856:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773118.568736:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773118.568775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773118.568815:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773118.632808:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773118.696801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773118.696822:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773118.696846:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773118.824793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773118.824802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773118.824830:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773118.824847:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773118.888841:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773118.888842:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773118.888858:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773118.888872:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773118.888940:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773118.889008:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773119.016778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773119.080772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773119.144781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773119.208883:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773119.208910:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773119.208911:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773119.208927:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773119.272834:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773119.272897:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773119.273135:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773119.400776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773119.400826:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773119.464811:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773119.528840:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773119.592790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773119.592791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773119.592845:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773119.656761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773119.720798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773119.720818:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773119.720863:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773119.848781:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773119.848809:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773119.848826:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773119.848855:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773119.912790:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773119.912790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773119.912851:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773119.912873:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773119.912894:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773119.912896:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773120.040818:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773120.104856:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773120.168747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773120.232797:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773120.232799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773120.232819:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773120.232847:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773120.296853:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773120.296946:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773120.297016:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773120.424784:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773120.424800:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773120.488792:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773120.552841:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773120.616800:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773120.616806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773120.616816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773120.680767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773120.744790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773120.744796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773120.744835:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773120.872778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773120.872785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773120.872826:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773120.872829:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773120.936838:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773120.936843:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773120.936845:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773120.936845:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773120.936893:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773120.936906:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773121.064774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773121.128766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773121.192824:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773121.192865:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773121.192877:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773121.192884:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773121.192905:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2117 00000100:00080000:3.0:1790773121.192924:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773121.192948:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773121.193026:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534604800/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773121.193081:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773121.193088:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773121.193091:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773121.193097:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2117 00000100:00080000:3.0:1790773121.193104:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773121.193115:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773121.193133:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773121.193145:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534604800:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773121.193162:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534604928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773121.193180:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773121.193186:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773121.193190:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773121.193196:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2117 00000100:00080000:3.0:1790773121.193202:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773121.193209:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773121.193217:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773121.193233:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534605056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773121.193247:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773121.193253:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773121.193255:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773121.193261:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2117 00000100:00080000:3.0:1790773121.193266:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773121.193272:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773121.193279:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773121.193293:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534605184/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773121.193305:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773121.193311:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773121.193314:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773121.193320:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2117 00000100:00080000:3.0:1790773121.193326:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773121.193332:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773121.193338:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773121.193351:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534605312/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773121.193366:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773121.193370:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534604928:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773121.193373:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773121.193380:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773121.193393:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2117 00000100:00080000:3.0:1790773121.193401:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773121.193408:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773121.193417:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773121.193437:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534605440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773121.193448:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534605056:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773121.193456:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773121.193463:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773121.193467:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773121.193476:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2117 00000100:00080000:3.0:1790773121.193482:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773121.193491:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773121.193495:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534605184:192.168.204.150@tcp:8:kworker.0 00000008:00100000:3.0:1790773121.193501:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773121.193525:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534605568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773121.193539:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773121.193547:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1790773121.193549:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534605312:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790773121.193551:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773121.193567:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2117 00000100:00080000:3.0:1790773121.193573:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773121.193581:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773121.193588:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773121.193599:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534605440:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773121.193610:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534605696/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773121.193628:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773121.193634:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773121.193637:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773121.193642:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534605568:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773121.193723:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2117 00000100:00080000:3.0:1790773121.193733:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773121.193746:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773121.193757:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773121.193779:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534605696:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773121.193786:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534605824/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773121.193827:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534605824:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773121.256809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773121.256818:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773121.256837:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773121.256842:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773121.320851:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773121.320864:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773121.320926:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773121.448784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773121.448810:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773121.512815:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773121.576847:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773121.640781:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773121.640813:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773121.640828:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773121.704823:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773121.768775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773121.768817:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773121.768825:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773121.896829:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773121.896832:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773121.896902:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773121.896912:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773121.960796:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773121.960824:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773121.960834:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773121.960860:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773121.960866:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773121.960905:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773122.088789:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773122.152761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773122.216791:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773122.280775:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773122.280817:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773122.280818:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773122.280819:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773122.344828:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773122.344861:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773122.344927:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773122.472800:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773122.472833:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773122.536790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773122.600800:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773122.664782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773122.664819:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773122.664821:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773122.728772:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773122.792784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773122.792801:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773122.792803:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773122.920763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773122.920785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773122.920803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773122.920806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773122.984793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773122.984821:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773122.984897:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773122.984920:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773122.984953:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773122.984954:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773123.112777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773123.176770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773123.240804:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773123.304802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773123.304868:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773123.304888:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773123.304889:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773123.368793:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773123.368877:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773123.368938:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773123.496764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773123.496821:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773123.560808:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773123.624770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773123.688820:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773123.688853:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773123.688855:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773123.752773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773123.816812:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773123.816867:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773123.816872:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773123.944777:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773123.944782:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773123.944815:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773123.944853:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773124.008767:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773124.008828:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773124.008853:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773124.008855:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773124.008867:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773124.008912:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773124.136737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773124.200738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773124.264779:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:1.0:1790773124.265105:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:1.0:1790773124.265171:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1790773124.265199:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773124.265282:0:2385:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:1.0:1790773124.265333:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773124.265350:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773121/real 1790773124] req@ffff9c4ef73c0000 x1877759534605824/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790773124.265365:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773124.265398:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773124.265441:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534605824/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790773124.265442:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773124.265473:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:0.0:1790773124.265487:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000800:00000100:1.0:1790773124.265516:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:0.0:1790773124.265526:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773124.265545:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534605824:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773124.265565:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773124.265585:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773121/real 1790773124] req@ffff9c4ef73c1500 x1877759534605312/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773124.265624:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534605312/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773124.265719:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773124.265729:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773124.265738:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534605312:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773124.265776:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773121/real 1790773124] req@ffff9c4ef73c0700 x1877759534605440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773124.265802:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534605440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773124.265841:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773124.265849:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773124.265857:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534605440:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773124.265872:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773121/real 1790773124] req@ffff9c4ef73c2a00 x1877759534605568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773124.265896:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534605568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773124.265913:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773124.265920:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773124.265927:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534605568:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773124.265943:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773121/real 1790773124] req@ffff9c4ef73c2d80 x1877759534604928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773124.265967:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534604928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773124.265994:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773124.266005:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773124.266012:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534604928:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773124.266026:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773121/real 1790773124] req@ffff9c4ef73c3800 x1877759534605056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773124.266047:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534605056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773124.266063:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773124.266069:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773124.266076:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534605056:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773124.266088:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773121/real 1790773124] req@ffff9c4ef73c0a80 x1877759534605184/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773124.266109:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534605184/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773124.266127:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773124.266133:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773124.266140:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534605184:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773124.266154:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773121/real 1790773124] req@ffff9c4ef73c0e00 x1877759534604800/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773124.266178:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534604800/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773124.266198:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773124.266212:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773124.266221:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534604800:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773124.266304:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773121/real 1790773124] req@ffff9c4ef73c2300 x1877759534605696/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773124.266332:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534605696/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773124.266354:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773124.266362:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773124.266370:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534605696:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773124.328768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773124.328791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773124.328811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773124.328818:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773124.392809:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773124.392893:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773124.392925:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773124.520769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773124.520777:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773124.584746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773124.648780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773124.712787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773124.712820:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773124.712833:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773124.776737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773124.840760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773124.840778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773124.840802:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773124.968833:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773124.968847:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773124.968871:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773124.968925:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773125.032762:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773125.032806:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773125.032848:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773125.032852:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773125.032879:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773125.032902:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773125.160768:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773125.224740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773125.288778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773125.352771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773125.352801:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773125.352801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773125.352823:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773125.416792:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773125.416816:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773125.417052:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773125.544804:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773125.544825:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773125.608768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773125.672781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773125.736738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773125.736840:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773125.736852:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773125.800780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773125.864872:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773125.864884:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773125.864896:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773125.992775:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773125.992815:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773125.992854:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773125.992885:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773126.056805:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773126.056806:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773126.056840:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773126.056864:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773126.056902:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773126.056939:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773126.184758:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773126.248750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773126.312714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773126.312816:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773126.312826:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773126.312833:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773126.312849:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2122 00000100:00080000:2.0:1790773126.312866:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773126.312885:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773126.312943:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534605952/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773126.312991:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773126.312997:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773126.313000:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773126.313005:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2122 00000100:00080000:2.0:1790773126.313010:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773126.313016:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773126.313030:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773126.313051:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534606080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773126.313067:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773126.313072:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773126.313075:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773126.313085:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2122 00000100:00100000:0.0:1790773126.313089:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534605952:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773126.313091:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773126.313104:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773126.313112:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773126.313131:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534606208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773126.313151:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773126.313160:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773126.313164:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773126.313174:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2122 00000100:00080000:2.0:1790773126.313183:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773126.313194:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773126.313206:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:0.0:1790773126.313218:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773126.313236:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534606336/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773126.313258:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773126.313267:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773126.313272:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773126.313283:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2122 00000100:00080000:2.0:1790773126.313292:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773126.313305:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773126.313317:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773126.313325:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773126/real 1790773126] req@ffff9c4efbcc0a80 x1877759534605952/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773126.313348:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534606464/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773126.313371:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773126.313377:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534605952/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773126.313381:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773126.313390:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773126.313408:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2122 00000100:00080000:2.0:1790773126.313423:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773126.313424:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773126.313441:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773126.313443:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773126.313461:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773126.313462:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534605952:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773126.313501:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534606592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773126.313504:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534606080:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773126.313540:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773126.313548:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773126.313553:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773126.313565:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2122 00000100:00080000:2.0:1790773126.313576:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773126.313583:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773126.313593:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773126.313635:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773126.313701:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773126/real 1790773126] req@ffff9c4efbcc0700 x1877759534606080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773126.313740:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534606080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773126.313775:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534606720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773126.313782:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773126.313804:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773126.313815:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773126.313821:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534606080:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773126.313827:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773126.313838:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773126.313850:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2122 00000100:00080000:2.0:1790773126.313860:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773126.313877:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773126.313889:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773126.313899:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534606208:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773126.313919:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534606848/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773126.313945:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773126.313963:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773126.313968:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773126.313976:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773126.313992:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2122 00000100:00080000:2.0:1790773126.314004:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773126.314016:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773126/real 1790773126] req@ffff9c4efbcc2680 x1877759534606208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773126.314020:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773126.314039:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773126.314061:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534606208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773126.314075:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534606976/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773126.314094:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773126.314112:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773126.314123:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534606208:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773126.314141:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534606336:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773126.314196:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773126.314223:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773126/real 1790773126] req@ffff9c4efbcc2300 x1877759534606336/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773126.314249:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534606336/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773126.314362:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773126.314372:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773126.314378:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534606336:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773126.314390:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534606464:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773126.314482:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773126.314518:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773126/real 1790773126] req@ffff9c4efbcc1880 x1877759534606464/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773126.314538:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534606464/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773126.314551:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773126.314556:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773126.314560:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534606464:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773126.314569:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534606592:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773126.314596:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773126.314631:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773126/real 1790773126] req@ffff9c4efbcc1500 x1877759534606592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773126.314645:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534606592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773126.314710:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773126.314715:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773126.314719:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534606592:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773126.314727:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534606720:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773126.314753:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773126.314767:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773126/real 1790773126] req@ffff9c4efbcc1f80 x1877759534606720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773126.314780:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534606720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773126.314791:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773126.314795:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773126.314799:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534606720:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773126.314842:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534606848:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773126.314886:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773126.314901:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773126/real 1790773126] req@ffff9c4efbcc1c00 x1877759534606848/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773126.314915:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534606848/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773126.314924:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773126.314933:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773126.314936:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534606848:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773126.314942:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534606976:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773126.314960:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773126.314969:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773126/real 1790773126] req@ffff9c4efbcc2a00 x1877759534606976/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773126.314979:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534606976/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773126.314986:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773126.314989:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773126.314991:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534606976:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773126.376764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773126.376787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773126.376787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773126.376792:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773126.440818:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773126.440895:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773126.440939:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773126.568741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773126.568788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773126.632756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773126.696729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773126.760768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773126.760777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773126.760778:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773126.824717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773126.888729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773126.888729:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773126.888787:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773127.016764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773127.016766:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773127.016798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773127.016810:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773127.080730:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773127.080730:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773127.080735:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773127.080779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773127.080794:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773127.080821:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773127.208699:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773127.272714:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773127.336777:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773127.400735:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773127.400751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773127.400754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773127.400785:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773127.464824:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773127.464830:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773127.464913:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773127.592752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773127.592818:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773127.656782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773127.720742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773127.784727:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773127.784761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773127.784761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773127.848745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773127.912760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773127.912760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773127.912760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773128.040723:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773128.040725:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773128.040726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773128.040736:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773128.104724:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773128.104740:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773128.104740:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773128.104747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773128.104768:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773128.104771:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773128.232730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773128.296751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773128.360713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773128.424741:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773128.424749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773128.424749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773128.424749:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773128.488728:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773128.488764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773128.488885:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773128.616742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773128.616742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773128.680750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773128.744748:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773128.808780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773128.808780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773128.808780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773128.872739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773128.936724:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773128.936724:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773128.936728:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773129.064756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773129.064756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773129.064756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773129.064756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773129.128725:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773129.128725:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773129.128730:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773129.128733:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773129.128758:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773129.128772:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773129.256705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773129.320719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773129.384699:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773129.448732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773129.448732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773129.448732:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773129.448732:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773129.512728:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773129.512740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773129.512752:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773129.640714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773129.640727:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773129.704739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773129.768769:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773129.832757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773129.832757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773129.832757:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773129.896730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773129.960751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773129.960751:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773129.960751:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773130.088715:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773130.088720:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773130.088725:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773130.088725:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773130.152739:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773130.152739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773130.152746:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773130.152748:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773130.152778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773130.152790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773130.280724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773130.344747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773130.408711:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773130.472777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773130.472777:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773130.472802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773130.472812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773130.536731:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773130.536747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773130.536850:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773130.664707:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773130.664709:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773130.728757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773130.792724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773130.856746:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773130.856765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773130.856766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773130.920709:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773130.984733:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773130.984739:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773130.984742:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773131.112757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773131.112757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773131.112758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773131.112768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773131.176727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773131.176759:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773131.176760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773131.176770:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773131.176786:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773131.176827:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773131.304730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773131.368759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773131.432724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773131.432751:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773131.432757:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773131.432761:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773131.432770:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2127 00000100:00080000:3.0:1790773131.432780:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773131.432791:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773131.432826:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534607104/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773131.432848:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773131.432852:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773131.432853:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773131.432857:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2127 00000100:00080000:3.0:1790773131.432859:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773131.432863:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773131.432871:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773131.432882:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534607232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773131.432890:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534607104:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773131.432906:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773131.432908:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773131.432910:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773131.432913:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2127 00000100:00080000:3.0:1790773131.432916:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773131.432919:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773131.432922:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773131.432932:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534607360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773131.432938:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773131.432941:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773131.432942:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773131.432945:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2127 00000100:00080000:3.0:1790773131.432949:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773131.432952:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773131.432955:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:0.0:1790773131.432963:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773131.432964:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534607488/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773131.432972:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773131.432975:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773131.432977:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773131.432980:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2127 00000100:00080000:3.0:1790773131.432984:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773131.432988:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773131.432991:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773131.432998:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534607616/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773131.433005:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773131.433007:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773131.433009:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773131.433014:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773131/real 1790773131] req@ffff9c4ef73c1c00 x1877759534607104/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773131.433015:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2127 00000100:00080000:3.0:1790773131.433019:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773131.433025:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773131.433032:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773131.433044:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534607744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773131.433051:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534607104/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773131.433052:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773131.433061:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773131.433065:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773131.433074:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773131.433075:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2127 00000100:00080000:3.0:1790773131.433083:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773131.433084:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773131.433093:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773131.433094:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534607104:192.168.204.150@tcp:250:kworker.0 00000008:00100000:3.0:1790773131.433103:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773131.433119:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534607872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773131.433125:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534607232:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773131.433141:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773131.433144:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773131.433146:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773131.433150:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2127 00000100:00080000:3.0:1790773131.433154:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773131.433161:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773131.433162:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790773131.433166:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773131.433177:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534608000/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773131.433183:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773131/real 1790773131] req@ffff9c4ef73c2300 x1877759534607232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773131.433187:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773131.433195:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773131.433198:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773131.433205:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2127 00000100:00080000:3.0:1790773131.433209:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773131.433214:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773131.433215:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534607232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773131.433218:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773131.433229:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534608128/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773131.433241:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773131.433247:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773131.433252:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534607232:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773131.433260:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534607360:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773131.433279:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773131.433291:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773131/real 1790773131] req@ffff9c4ef73c0e00 x1877759534607360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773131.433303:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534607360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773131.433312:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773131.433316:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773131.433319:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534607360:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773131.433325:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534607488:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773131.433343:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773131.433363:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773131/real 1790773131] req@ffff9c4ef73c0a80 x1877759534607488/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773131.433382:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534607488/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773131.433394:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773131.433398:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773131.433402:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534607488:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773131.433409:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534607616:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773131.433429:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773131.433440:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773131/real 1790773131] req@ffff9c4ef73c3800 x1877759534607616/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773131.433452:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534607616/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773131.433461:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773131.433465:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773131.433468:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534607616:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773131.433475:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534607744:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773131.433491:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773131.433518:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773131/real 1790773131] req@ffff9c4ef73c2d80 x1877759534607744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773131.433531:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534607744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773131.433547:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773131.433551:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773131.433555:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534607744:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773131.433578:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534607872:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773131.433601:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773131.433614:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773131/real 1790773131] req@ffff9c4ef73c2a00 x1877759534607872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773131.433627:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534607872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773131.433636:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773131.433640:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773131.433644:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534607872:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773131.433672:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534608000:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773131.433693:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773131.433705:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773131/real 1790773131] req@ffff9c4ef73c0700 x1877759534608000/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773131.433717:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534608000/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773131.433727:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773131.433731:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773131.433734:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534608000:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773131.433741:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534608128:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773131.433759:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773131.433771:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773131/real 1790773131] req@ffff9c4ef73c1500 x1877759534608128/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773131.433785:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534608128/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773131.433794:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773131.433798:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773131.433802:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534608128:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773131.496752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773131.496765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773131.496775:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773131.496780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773131.560729:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773131.560788:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773131.560861:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773131.688721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773131.688721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773131.752757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773131.816705:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773131.880719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773131.880724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773131.880762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773131.944723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773132.008733:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773132.008736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773132.008738:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773132.136736:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773132.136736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773132.136739:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773132.136745:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773132.200746:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773132.200749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773132.200750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773132.200765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773132.200781:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773132.200782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773132.328704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773132.392710:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773132.456712:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773132.520724:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773132.520724:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773132.520726:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773132.520729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773132.584737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773132.584754:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773132.584778:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773132.712728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773132.712728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773132.776731:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773132.840738:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773132.904741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773132.904741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773132.904741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773132.968740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773133.032730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773133.032731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773133.032735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773133.160758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773133.160759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773133.160759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773133.160759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773133.224712:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773133.224717:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773133.224736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773133.224750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773133.224760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773133.224770:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773133.352722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773133.416744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773133.480725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773133.544731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773133.544740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773133.544771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773133.544773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773133.608747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773133.608836:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773133.608870:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773133.736730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773133.736737:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773133.800730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773133.864728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773133.928725:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773133.928735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773133.928766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773133.992751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773134.056745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773134.056745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773134.056756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773134.184745:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773134.184745:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773134.184747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773134.184787:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773134.248726:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773134.248748:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773134.248751:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773134.248770:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773134.248786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773134.248802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773134.376701:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773134.440776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773134.504726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773134.568768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773134.568768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773134.568769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773134.568775:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773134.632729:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773134.632774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773134.632834:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773134.760741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773134.760741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773134.824757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773134.888718:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773134.952755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773134.952766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773134.952773:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773135.016705:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773135.080737:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773135.080738:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773135.080769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773135.208737:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773135.208737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773135.208761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773135.208769:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773135.272714:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773135.272732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773135.272756:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773135.272761:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773135.272762:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773135.272780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773135.400740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773135.464738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773135.528699:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773135.592748:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773135.592748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773135.592791:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773135.592796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773135.656745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773135.656793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773135.656897:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773135.784707:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773135.784707:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773135.848723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773135.912742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773135.976737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773135.976748:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773135.976774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773136.040719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773136.104769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773136.104777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773136.104778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773136.232716:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773136.232717:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773136.232728:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773136.232730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773136.296718:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773136.296728:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773136.296734:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773136.296750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773136.296767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773136.296769:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773136.424704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773136.488732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773136.552704:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773136.552745:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773136.552750:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773136.552755:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773136.552763:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2132 00000100:00080000:1.0:1790773136.552772:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773136.552781:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773136.552813:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534608256/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773136.552836:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773136.552839:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773136.552840:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773136.552844:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2132 00000100:00080000:1.0:1790773136.552847:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773136.552850:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773136.552859:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773136.552871:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534608384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773136.552874:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534608256:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773136.552887:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773136.552890:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773136.552891:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773136.552895:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2132 00000100:00080000:1.0:1790773136.552898:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773136.552900:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773136.552905:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773136.552915:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534608512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773136.552923:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773136.552925:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773136.552927:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773136.552929:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2132 00000100:00080000:1.0:1790773136.552933:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773136.552937:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773136.552941:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773136.552946:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773136.552956:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534608640/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773136.552962:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773136.552965:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773136.552967:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773136.552971:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2132 00000100:00080000:1.0:1790773136.552975:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773136.552978:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773136/real 1790773136] req@ffff9c4efac4fb80 x1877759534608256/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773136.552981:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773136.552988:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773136.553000:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534608768/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773136.553004:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534608256/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773136.553016:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773136.553020:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773136.553022:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773136.553023:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773136.553029:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773136.553031:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2132 00000100:00080000:1.0:1790773136.553034:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773136.553036:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534608256:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773136.553041:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773136.553045:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773136.553053:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534608896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773136.553060:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773136.553061:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534608384:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773136.553068:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773136.553071:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773136.553076:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2132 00000100:00080000:1.0:1790773136.553079:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773136.553085:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773136.553089:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773136.553093:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773136.553107:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773136/real 1790773136] req@ffff9c4efac4df80 x1877759534608384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773136.553107:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534609024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773136.553143:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773136.553147:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1790773136.553148:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534608384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773136.553149:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773136.553158:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2132 00000100:00080000:1.0:1790773136.553162:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773136.553167:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773136.553170:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773136.553171:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773136.553177:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773136.553179:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534608384:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773136.553184:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534608512:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773136.553189:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534609152/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773136.553197:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773136.553199:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773136.553200:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773136.553203:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2132 00000100:00080000:1.0:1790773136.553207:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773136.553212:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773136.553215:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773136.553221:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773136.553226:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773136/real 1790773136] req@ffff9c4efac4dc00 x1877759534608512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773136.553233:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534609280/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773136.553242:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534608512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773136.553250:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773136.553253:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773136.553257:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534608512:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773136.553262:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534608640:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773136.553275:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773136.553282:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773136/real 1790773136] req@ffff9c4efac4ed80 x1877759534608640/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773136.553294:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534608640/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773136.553301:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773136.553303:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773136.553305:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534608640:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773136.553309:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534608768:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773136.553320:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773136.553326:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773136/real 1790773136] req@ffff9c4efac4c700 x1877759534608768/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773136.553333:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534608768/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773136.553340:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773136.553342:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773136.553344:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534608768:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773136.553355:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534608896:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773136.553367:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773136.553373:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773136/real 1790773136] req@ffff9c4efac4f800 x1877759534608896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773136.553380:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534608896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773136.553396:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773136.553398:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773136.553418:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534608896:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773136.553423:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534609024:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773136.553438:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773136.553446:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773136/real 1790773136] req@ffff9c4efac4f480 x1877759534609024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773136.553454:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534609024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773136.553460:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773136.553462:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773136.553464:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534609024:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773136.553468:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534609152:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773136.553477:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773136.553483:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773136/real 1790773136] req@ffff9c4efac4e680 x1877759534609152/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773136.553491:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534609152/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773136.553498:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773136.553500:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773136.553503:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534609152:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773136.553507:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534609280:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773136.553517:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773136.553522:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773136/real 1790773136] req@ffff9c4efac4ea00 x1877759534609280/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773136.553529:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534609280/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773136.553536:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773136.553538:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773136.553540:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534609280:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773136.616745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773136.616745:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773136.616745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773136.616746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773136.680724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773136.680746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773136.680770:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773136.808722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773136.808722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773136.872712:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773136.936734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773137.000720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773137.000720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773137.000724:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773137.064717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773137.128736:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773137.128736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773137.128740:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773137.256718:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773137.256723:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773137.256725:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773137.256726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773137.320720:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773137.320725:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773137.320743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773137.320784:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773137.320806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773137.320811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773137.448707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773137.512706:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773137.576704:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773137.640730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773137.640756:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773137.640757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773137.640757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773137.704750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773137.704777:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773137.704856:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773137.832721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773137.832721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773137.896708:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773137.960734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773138.024725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773138.024757:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773138.024770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773138.088711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773138.152730:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773138.152730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773138.152747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773138.280714:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773138.280715:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773138.280721:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773138.280767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773138.344725:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773138.344731:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773138.344751:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773138.344772:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773138.344774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773138.344776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773138.472705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773138.536711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773138.600732:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773138.664757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773138.664758:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773138.664758:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773138.664758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773138.728730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773138.728730:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773138.728762:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773138.856713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773138.856728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773138.920729:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773138.984715:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773139.048724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773139.048726:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773139.048726:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773139.112718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773139.176716:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773139.176719:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773139.176722:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773139.304720:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773139.304720:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773139.304726:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773139.304728:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773139.368708:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773139.368732:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773139.368738:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773139.368743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773139.368755:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773139.368767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773139.496709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773139.560706:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773139.624719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773139.688711:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773139.688720:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773139.688721:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773139.688725:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773139.752729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773139.752735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773139.752775:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773139.880712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773139.880724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773139.944708:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773140.008736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773140.072699:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773140.072719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773140.072719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773140.136698:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773140.200725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773140.200727:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773140.200728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773140.328708:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773140.328710:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773140.328724:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773140.328725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773140.392724:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773140.392725:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773140.392736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773140.392737:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773140.392761:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773140.392766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773140.520704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773140.584711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773140.648727:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773140.712728:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773140.712733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773140.712735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773140.712740:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773140.776716:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773140.776722:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773140.776847:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773140.904715:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773140.904715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773140.968777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773141.032710:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773141.096708:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773141.096713:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773141.096719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773141.160809:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773141.224715:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773141.224717:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773141.224723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773141.352726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773141.352727:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773141.352731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773141.352732:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773141.416724:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773141.416725:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773141.416729:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773141.416730:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773141.416755:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773141.416759:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773141.544698:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773141.608720:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773141.672714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773141.672725:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773141.672730:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773141.672735:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773141.672743:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2137 00000100:00080000:1.0:1790773141.672752:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773141.672761:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773141.672802:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534609408/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773141.672825:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773141.672828:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773141.672829:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773141.672833:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2137 00000100:00080000:1.0:1790773141.672836:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773141.672840:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773141.672848:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773141.672861:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534609536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773141.672862:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534609408:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773141.672876:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773141.672879:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773141.672880:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773141.672884:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2137 00000100:00080000:1.0:1790773141.672886:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773141.672890:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773141.672894:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773141.672903:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534609664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773141.672910:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773141.672913:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773141.672914:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773141.672917:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2137 00000100:00080000:1.0:1790773141.672922:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773141.672925:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773141.672928:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:0.0:1790773141.672938:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773141.672938:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534609792/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773141.672948:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773141.672950:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773141.672952:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773141.672956:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2137 00000100:00080000:1.0:1790773141.672965:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773141.672968:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773141.672971:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773141.672980:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534609920/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773141.672987:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773141.672989:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773141/real 1790773141] req@ffff9c4efac4e300 x1877759534609408/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773141.672990:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773141.672993:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773141.672999:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2137 00000100:00080000:1.0:1790773141.673003:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773141.673009:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773141.673013:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773141.673023:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534609408/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773141.673023:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534610048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773141.673046:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773141.673052:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773141.673055:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773141.673062:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2137 00000100:00080000:1.0:1790773141.673067:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773141.673073:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773141.673075:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773141.673077:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773141.673084:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773141.673091:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534610176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773141.673093:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534609408:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773141.673105:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773141.673114:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773141.673117:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773141.673123:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2137 00000100:00080000:1.0:1790773141.673128:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773141.673133:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773141.673139:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534609536:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790773141.673139:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773141.673153:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534610304/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773141.673162:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773141.673165:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773141.673167:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773141.673170:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2137 00000100:00080000:1.0:1790773141.673173:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773141.673180:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773141.673181:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773141.673184:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773141.673194:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534610432/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773141.673209:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773141/real 1790773141] req@ffff9c4efac4ea00 x1877759534609536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773141.673229:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534609536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773141.673262:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773141.673268:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773141.673272:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534609536:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773141.673281:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534609664:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773141.673323:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773141.673353:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773141/real 1790773141] req@ffff9c4efac4e680 x1877759534609664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773141.673369:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534609664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773141.673379:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773141.673382:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773141.673387:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534609664:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773141.673393:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534609792:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773141.673414:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773141.673426:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773141/real 1790773141] req@ffff9c4efac4f480 x1877759534609792/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773141.673439:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534609792/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773141.673449:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773141.673452:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773141.673456:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534609792:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773141.673462:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534609920:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773141.673483:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773141.673496:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773141/real 1790773141] req@ffff9c4efac4f800 x1877759534609920/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773141.673509:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534609920/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773141.673520:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773141.673524:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773141.673528:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534609920:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773141.673534:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534610048:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773141.673554:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773141.673565:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773141/real 1790773141] req@ffff9c4efac4c700 x1877759534610048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773141.673578:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534610048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773141.673594:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773141.673599:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773141.673603:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534610048:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773141.673609:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534610176:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773141.673629:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773141.673640:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773141/real 1790773141] req@ffff9c4efac4ed80 x1877759534610176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773141.673678:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534610176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773141.673688:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773141.673692:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773141.673696:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534610176:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773141.673715:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534610304:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773141.673731:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773141.673739:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773141/real 1790773141] req@ffff9c4efac4dc00 x1877759534610304/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773141.673747:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534610304/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773141.673760:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773141.673762:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773141.673764:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534610304:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773141.673768:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534610432:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773141.673782:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773141.673792:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773141/real 1790773141] req@ffff9c4efac4df80 x1877759534610432/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773141.673804:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534610432/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773141.673815:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773141.673819:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773141.673823:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534610432:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773141.736722:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773141.736730:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773141.736730:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773141.736745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773141.800718:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773141.800765:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773141.800780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773141.928705:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773141.928715:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773141.992728:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773142.056734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773142.120702:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773142.120714:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773142.120715:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773142.184716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773142.248726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773142.248728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773142.248751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773142.376774:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773142.376790:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773142.376823:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773142.376827:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773142.440770:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773142.440776:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773142.440817:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773142.440822:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773142.440869:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773142.440871:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773142.568746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773142.632765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773142.696864:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773142.760789:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773142.760851:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773142.760851:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773142.760851:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773142.824821:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773142.824828:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773142.824895:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773142.952774:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773142.952795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773143.016735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773143.080705:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773143.144699:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773143.144704:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773143.144721:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773143.208710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773143.272709:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773143.272733:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773143.272733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773143.400707:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773143.400713:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773143.400723:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773143.400725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773143.464713:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773143.464732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773143.464736:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773143.464736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773143.464766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773143.464775:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773143.592773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773143.656750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773143.720842:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773143.784753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773143.784791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773143.784804:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773143.784805:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773143.848777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773143.848791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773143.848853:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773143.976766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773143.976792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773144.040745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773144.104739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773144.168704:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773144.168704:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773144.168736:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773144.232709:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773144.296718:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773144.296720:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773144.296736:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773144.424712:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773144.424719:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773144.424721:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773144.424723:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773144.488758:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773144.488759:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773144.488759:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773144.488764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773144.488792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773144.488796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773144.616711:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773144.680711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773144.744722:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773144.808741:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773144.808741:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773144.808741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773144.808766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773144.872715:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773144.872731:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773144.872764:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773145.000726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773145.000726:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773145.064758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773145.128703:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773145.192699:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773145.192701:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773145.192769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773145.256706:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773145.320722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773145.320723:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773145.320777:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773145.448705:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773145.448722:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773145.448725:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773145.448769:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773145.512722:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773145.512722:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773145.512743:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773145.512777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773145.512782:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773145.512800:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773145.640704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773145.704715:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773145.768704:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773145.832716:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773145.832724:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773145.832746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773145.832776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773145.896726:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773145.896763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773145.896882:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773146.024710:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773146.024714:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773146.088748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773146.152731:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773146.216744:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773146.216745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773146.216800:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773146.280727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773146.344707:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773146.344713:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773146.344770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773146.472731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773146.472731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773146.472781:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773146.472792:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773146.536718:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773146.536726:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773146.536745:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773146.536773:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773146.536794:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773146.536796:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773146.664709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773146.728727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773146.792702:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773146.792752:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773146.792757:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773146.792761:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773146.792769:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2142 00000100:00080000:2.0:1790773146.792777:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773146.792786:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773146.792826:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534610560/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773146.792855:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773146.792858:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773146.792860:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773146.792864:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2142 00000100:00080000:2.0:1790773146.792868:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773146.792874:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773146.792883:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773146.792884:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534610560:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773146.792899:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534610688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773146.792908:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773146.792911:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773146.792913:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773146.792916:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2142 00000100:00080000:2.0:1790773146.792920:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773146.792926:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773146.792932:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773146.792944:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773146.792948:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534610816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773146.792962:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773146.792967:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773146.792969:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773146.792974:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2142 00000100:00080000:2.0:1790773146.792978:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773146.792982:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773146.792988:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773146.792991:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773146/real 1790773146] req@ffff9c4efbcc3100 x1877759534610560/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773146.793001:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534610944/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773146.793010:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534610560/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773146.793010:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773146.793018:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773146.793020:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773146.793027:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2142 00000100:00080000:2.0:1790773146.793030:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773146.793035:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773146.793037:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790773146.793039:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:0.0:1790773146.793044:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773146.793049:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534611072/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773146.793050:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534610560:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773146.793064:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773146.793066:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773146.793069:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773146.793072:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2142 00000100:00080000:2.0:1790773146.793075:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773146.793080:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534610688:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773146.793080:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773146.793087:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773146.793098:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534611200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773146.793106:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773146.793109:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773146.793112:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773146.793116:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2142 00000800:00000100:0.0:1790773146.793118:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773146.793120:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773146.793125:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773146.793130:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773146.793145:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773146/real 1790773146] req@ffff9c4efbcc0380 x1877759534610688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773146.793145:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534611328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773146.793166:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773146.793170:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773146.793172:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773146.793176:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534610688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773146.793180:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2142 00000100:00080000:2.0:1790773146.793193:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773146.793198:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773146.793211:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773146.793220:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773146.793223:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773146.793230:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534610688:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773146.793233:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534611456/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773146.793244:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773146.793246:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534610816:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773146.793248:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773146.793251:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773146.793256:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2142 00000100:00080000:2.0:1790773146.793260:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773146.793264:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773146.793268:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:0.0:1790773146.793277:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773146.793280:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534611584/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773146.793291:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773146/real 1790773146] req@ffff9c4efbcc2a00 x1877759534610816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773146.793303:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534610816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773146.793310:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773146.793313:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773146.793316:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534610816:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773146.793320:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534610944:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773146.793333:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773146.793343:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773146/real 1790773146] req@ffff9c4efbcc1c00 x1877759534610944/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773146.793356:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534610944/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773146.793368:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773146.793380:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773146.793384:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534610944:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773146.793391:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534611072:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773146.793410:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773146.793422:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773146/real 1790773146] req@ffff9c4efbcc1f80 x1877759534611072/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773146.793435:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534611072/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773146.793456:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773146.793459:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773146.793462:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534611072:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773146.793479:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534611200:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773146.793495:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773146.793503:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773146/real 1790773146] req@ffff9c4efbcc1500 x1877759534611200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773146.793516:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534611200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773146.793527:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773146.793529:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773146.793532:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534611200:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773146.793536:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534611328:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773146.793546:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773146.793552:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773146/real 1790773146] req@ffff9c4efbcc1880 x1877759534611328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773146.793559:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534611328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773146.793565:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773146.793567:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773146.793569:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534611328:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773146.793573:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534611456:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773146.793582:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773146.793588:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773146/real 1790773146] req@ffff9c4efbcc2300 x1877759534611456/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773146.793596:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534611456/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773146.793607:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773146.793610:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773146.793612:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534611456:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773146.793616:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534611584:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773146.793625:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773146.793631:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773146/real 1790773146] req@ffff9c4efbcc2680 x1877759534611584/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773146.793638:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534611584/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773146.793645:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773146.793647:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773146.793667:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534611584:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773146.856742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773146.856744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773146.856744:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773146.856796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773146.920756:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773146.920757:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773146.920802:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773147.048710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773147.048713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773147.112778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773147.176723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773147.240705:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773147.240722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773147.240760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773147.304701:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773147.368740:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773147.368740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773147.368766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773147.496725:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773147.496731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773147.496733:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773147.496779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773147.560733:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773147.560733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773147.560741:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773147.560765:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773147.560788:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773147.560797:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773147.688712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773147.752725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773147.816705:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773147.880716:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773147.880721:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773147.880721:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773147.880755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773147.944707:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773147.944717:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773147.944787:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773148.072710:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773148.072715:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773148.136721:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773148.200730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773148.264726:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773148.264727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773148.264728:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773148.328712:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773148.392711:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773148.392722:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773148.392727:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773148.520719:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773148.520720:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773148.520722:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773148.520736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773148.584718:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773148.584732:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773148.584734:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773148.584734:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773148.584757:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773148.584761:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773148.712712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773148.776714:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773148.840717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773148.904734:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773148.904734:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773148.904734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773148.904735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773148.968727:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773148.968741:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773148.968800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773149.096710:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773149.096732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773149.160717:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773149.224721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773149.288726:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773149.288727:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773149.288735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773149.352707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773149.416732:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773149.416732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773149.416732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773149.544708:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773149.544711:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773149.544722:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773149.544733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773149.608732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773149.608734:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773149.608740:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773149.608740:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773149.608767:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773149.608771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773149.736711:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773149.800712:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773149.864702:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773149.928720:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773149.928723:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773149.928726:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773149.928730:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773149.992705:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773149.992712:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773149.992771:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773150.120704:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773150.120718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773150.184767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773150.248702:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773150.312746:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773150.312746:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773150.312761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773150.376721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773150.440739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773150.440739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773150.440754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773150.568716:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773150.568717:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773150.568727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773150.568767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773150.632714:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773150.632721:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773150.632727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773150.632729:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773150.632759:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773150.632760:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773150.760717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773150.824718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773150.888707:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773150.952719:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773150.952728:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773150.952728:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773150.952777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773151.016716:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773151.016732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773151.016847:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773151.144707:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773151.144713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773151.208741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773151.272714:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773151.336733:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773151.336734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773151.336761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773151.400707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773151.464799:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773151.464804:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773151.464815:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773151.592718:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773151.592728:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773151.592753:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773151.592775:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773151.656718:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773151.656749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773151.656766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773151.656767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773151.656785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773151.656787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773151.784717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773151.848700:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773151.912706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773151.912755:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773151.912766:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773151.912772:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773151.912786:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2147 00000100:00080000:1.0:1790773151.912799:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773151.912812:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773151.912860:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534611712/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773151.912888:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773151.912891:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773151.912893:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773151.912897:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2147 00000100:00080000:1.0:1790773151.912900:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773151.912904:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773151.912916:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773151.912928:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534611712:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773151.912931:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534611840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773151.912948:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773151.912953:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773151.912956:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773151.912962:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2147 00000100:00080000:1.0:1790773151.912984:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773151.912991:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773151.912994:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773151.913002:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773151.913020:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534611968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773151.913031:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773151.913035:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773151.913037:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773151.913039:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773151/real 1790773151] req@ffff9c4efac4fb80 x1877759534611712/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773151.913044:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2147 00000100:00080000:1.0:1790773151.913057:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773151.913073:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773151.913078:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534611712/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773151.913079:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00080000:0.0:1790773151.913094:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773151.913113:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773151.913119:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534612096/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773151.913125:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534611712:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773151.913148:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773151.913150:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773151.913154:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773151.913158:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2147 00000100:00080000:1.0:1790773151.913162:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773151.913166:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534611840:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773151.913174:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773151.913182:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:0.0:1790773151.913201:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773151.913201:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534612224/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773151.913222:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773151.913223:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773151/real 1790773151] req@ffff9c4efac4df80 x1877759534611840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773151.913238:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773151.913245:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773151.913255:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2147 00000100:00080000:1.0:1790773151.913258:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773151.913276:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773151.913280:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534611840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773151.913284:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773151.913304:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534612352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773151.913311:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773151.913312:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773151.913324:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773151.913329:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773151.913331:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773151.913336:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534611840:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773151.913350:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2147 00000100:00080000:1.0:1790773151.913354:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773151.913355:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534611968:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773151.913364:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773151.913370:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773151.913383:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534612480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773151.913389:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773151.913404:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773151/real 1790773151] req@ffff9c4efac4dc00 x1877759534611968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773151.913423:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773151.913433:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773151.913435:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773151.913442:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534611968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773151.913446:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2147 00000100:00080000:1.0:1790773151.913457:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773151.913470:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773151.913470:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773151.913485:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790773151.913487:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773151.913495:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534611968:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773151.913503:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534612608/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773151.913507:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534612096:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773151.913532:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773151.913535:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773151.913537:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773151.913545:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2147 00000800:00000100:0.0:1790773151.913547:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773151.913554:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773151.913566:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773151.913567:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773151/real 1790773151] req@ffff9c4efac4ed80 x1877759534612096/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773151.913578:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773151.913597:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534612096/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773151.913602:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534612736/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773151.913681:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773151.913685:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773151.913689:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534612096:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773151.913724:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534612224:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773151.913741:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773151.913749:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773151/real 1790773151] req@ffff9c4efac4c700 x1877759534612224/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773151.913757:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534612224/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773151.913764:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773151.913767:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773151.913769:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534612224:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773151.913773:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534612352:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773151.913783:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773151.913789:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773151/real 1790773151] req@ffff9c4efac4f800 x1877759534612352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773151.913798:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534612352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773151.913809:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773151.913811:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773151.913813:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534612352:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773151.913817:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534612480:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773151.913827:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773151.913834:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773151/real 1790773151] req@ffff9c4efac4f480 x1877759534612480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773151.913842:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534612480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773151.913847:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773151.913850:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773151.913852:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534612480:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773151.913856:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534612608:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773151.913866:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773151.913872:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773151/real 1790773151] req@ffff9c4efac4e680 x1877759534612608/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773151.913879:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534612608/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773151.913891:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773151.913893:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773151.913895:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534612608:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773151.913899:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534612736:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773151.913909:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773151.913915:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773151/real 1790773151] req@ffff9c4efac4ea00 x1877759534612736/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773151.913922:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534612736/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773151.913928:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773151.913930:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773151.913932:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534612736:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773151.976707:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773151.976725:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773151.976726:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773151.976757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773152.040730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773152.040771:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773152.040835:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773152.168718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773152.168722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773152.232769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773152.296709:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773152.360762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773152.360798:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773152.360803:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773152.424715:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773152.488773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773152.488778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773152.488778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773152.616720:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773152.616720:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773152.616748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773152.616777:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773152.680721:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773152.680763:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773152.680766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773152.680770:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773152.680792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773152.680803:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773152.808707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773152.872764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773152.936716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773153.000723:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773153.000724:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773153.000734:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773153.000763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773153.064760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773153.064760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773153.064830:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773153.192741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773153.192751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773153.256738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773153.320736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773153.384714:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773153.384721:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773153.384727:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773153.448735:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773153.512718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773153.512730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773153.512735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773153.640744:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773153.640744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773153.640769:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773153.640780:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773153.704745:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773153.704750:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773153.704768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773153.704776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773153.704795:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773153.704811:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773153.832710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773153.896722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773153.960709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773154.024737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773154.024739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773154.024739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773154.024742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773154.088710:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773154.088712:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773154.088757:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773154.216730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773154.216732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773154.280727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773154.344719:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773154.408736:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773154.408736:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773154.408737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773154.472732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773154.536713:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773154.536730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773154.536730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773154.664759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773154.664759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773154.664759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773154.664759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773154.728727:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773154.728736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773154.728745:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773154.728750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773154.728772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773154.728777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773154.856727:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773154.920755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773154.984727:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773155.048724:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773155.048729:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773155.048734:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773155.048734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773155.112741:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773155.112741:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773155.112793:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773155.240715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773155.240721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773155.304734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773155.368723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773155.432736:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773155.432736:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773155.432736:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773155.496747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773155.560742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773155.560743:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773155.560743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773155.688741:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773155.688741:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773155.688741:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773155.688743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773155.752724:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773155.752730:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773155.752731:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773155.752753:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773155.752771:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773155.752784:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773155.880715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773155.944732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773156.008723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773156.072764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773156.072764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773156.072766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773156.072766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773156.136730:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773156.136731:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773156.136775:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773156.264729:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773156.264730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773156.328727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773156.392723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773156.456735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773156.456735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773156.456736:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773156.520715:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773156.584741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773156.584741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773156.584741:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773156.712756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773156.712756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773156.712756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773156.712765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773156.776714:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773156.776718:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773156.776733:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773156.776745:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773156.776758:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773156.776768:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773156.904732:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773156.968759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773157.032731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773157.032735:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773157.032742:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773157.032747:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773157.032755:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2152 00000100:00080000:1.0:1790773157.032765:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773157.032775:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773157.032810:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534612864/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773157.032831:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773157.032834:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773157.032836:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773157.032839:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2152 00000100:00080000:1.0:1790773157.032841:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773157.032845:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773157.032853:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773157.032866:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534612864:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773157.032867:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534612992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773157.032881:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773157.032884:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773157.032886:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773157.032890:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2152 00000100:00080000:1.0:1790773157.032892:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773157.032895:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773157.032899:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773157.032907:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534613120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773157.032919:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773157.032922:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773157.032923:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773157.032926:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2152 00000100:00080000:1.0:1790773157.032929:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773157.032934:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773157.032937:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773157.032941:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773157.032950:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534613248/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773157.032957:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773157.032960:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773157.032961:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773157.032965:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2152 00000100:00080000:1.0:1790773157.032968:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773157.032971:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773157.032974:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773157.032975:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773157/real 1790773157] req@ffff9c4efac4e300 x1877759534612864/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773157.032985:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534613376/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773157.032994:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534612864/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773157.032995:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773157.033017:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773157.033021:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773157.033028:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2152 00000100:00080000:0.0:1790773157.033031:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773157.033032:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773157.033040:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773157.033042:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773157.033048:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534612864:192.168.204.150@tcp:250:kworker.0 00000008:00100000:1.0:1790773157.033048:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773157.033063:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534613504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773157.033070:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773157.033073:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1790773157.033076:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534612992:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773157.033077:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773157.033087:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2152 00000100:00080000:1.0:1790773157.033090:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773157.033093:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773157.033097:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773157.033108:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773157.033109:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534613632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773157.033120:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773157.033124:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773157/real 1790773157] req@ffff9c4efac4ea00 x1877759534612992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773157.033125:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773157.033128:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773157.033136:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2152 00000100:00080000:1.0:1790773157.033141:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773157.033145:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534612992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773157.033147:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773157.033153:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773157.033163:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534613760/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773157.033172:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773157.033175:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773157.033180:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773157.033181:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773157.033182:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773157.033191:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2152 00000100:00100000:0.0:1790773157.033194:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534612992:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773157.033195:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773157.033202:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773157.033206:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773157.033210:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534613120:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773157.033215:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534613888/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773157.033240:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773157.033250:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773157/real 1790773157] req@ffff9c4efac4e680 x1877759534613120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773157.033259:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534613120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773157.033265:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773157.033268:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773157.033270:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534613120:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773157.033275:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534613248:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773157.033287:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773157.033293:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773157/real 1790773157] req@ffff9c4efac4f480 x1877759534613248/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773157.033301:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534613248/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773157.033308:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773157.033310:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773157.033312:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534613248:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773157.033315:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534613376:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773157.033325:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773157.033331:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773157/real 1790773157] req@ffff9c4efac4f800 x1877759534613376/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773157.033339:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534613376/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773157.033345:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773157.033347:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773157.033349:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534613376:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773157.033353:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534613504:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773157.033363:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773157.033369:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773157/real 1790773157] req@ffff9c4efac4c700 x1877759534613504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773157.033377:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534613504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773157.033388:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773157.033390:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773157.033392:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534613504:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773157.033405:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534613632:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773157.033417:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773157.033423:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773157/real 1790773157] req@ffff9c4efac4ed80 x1877759534613632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773157.033431:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534613632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773157.033436:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773157.033438:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773157.033440:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534613632:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773157.033444:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534613760:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773157.033454:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773157.033460:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773157/real 1790773157] req@ffff9c4efac4dc00 x1877759534613760/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773157.033467:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534613760/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773157.033479:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773157.033482:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773157.033484:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534613760:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773157.033487:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534613888:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773157.033497:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773157.033502:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773157/real 1790773157] req@ffff9c4efac4df80 x1877759534613888/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773157.033510:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534613888/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773157.033516:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773157.033518:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773157.033521:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534613888:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773157.096724:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773157.096725:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773157.096725:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773157.096730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773157.160728:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773157.160729:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773157.160776:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773157.288723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773157.288723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773157.352736:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773157.416720:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773157.480727:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773157.480729:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773157.480732:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773157.544722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773157.608739:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773157.608739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773157.608739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773157.736747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773157.736747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773157.736747:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773157.736747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773157.800713:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773157.800735:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773157.800738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773157.800748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773157.800755:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773157.800765:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773157.928704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773157.992719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773158.056732:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773158.120765:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773158.120765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773158.120765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773158.120765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773158.184713:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773158.184727:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773158.184835:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773158.312730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773158.312748:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773158.376807:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773158.440732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773158.504714:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773158.504715:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773158.504731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773158.568710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773158.632720:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773158.632728:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773158.632728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773158.760734:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773158.760734:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773158.760737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773158.760760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773158.824719:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773158.824725:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773158.824731:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773158.824734:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773158.824760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773158.824765:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773158.952713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773159.016737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773159.080698:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773159.144703:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773159.144728:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773159.144729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773159.144730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773159.208721:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773159.208722:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773159.208808:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773159.336719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773159.336720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773159.400718:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773159.464729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773159.528709:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773159.528709:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773159.528725:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773159.592722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773159.656727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773159.656734:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773159.656734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773159.784711:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773159.784729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773159.784733:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773159.784735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773159.848718:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773159.848724:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773159.848726:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773159.848732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773159.848754:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773159.848763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773159.976718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773160.040724:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773160.104717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773160.168751:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773160.168751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773160.168751:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773160.168751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773160.232712:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773160.232713:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773160.232820:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773160.360733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773160.360736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773160.424775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773160.488721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773160.552713:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773160.552719:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773160.552753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773160.616714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773160.680741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773160.680789:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773160.680789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773160.808757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773160.808758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773160.808762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773160.808849:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773160.872721:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773160.872735:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773160.872756:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773160.872773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773160.872818:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773160.872824:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773161.000724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773161.064751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773161.128713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773161.192749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773161.192790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773161.192813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773161.192820:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773161.256736:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773161.256740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773161.256897:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773161.384719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773161.384719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773161.448755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773161.512734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773161.576719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773161.576731:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773161.576760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773161.640714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773161.704737:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773161.704751:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773161.704766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773161.832726:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773161.832732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773161.832743:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773161.832760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773161.896702:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773161.896721:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773161.896758:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773161.896761:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773161.896763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773161.896764:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773162.024739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773162.088731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773162.152756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773162.152788:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773162.152804:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773162.152812:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773162.152828:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2158 00000100:00080000:1.0:1790773162.152845:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773162.152864:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773162.152965:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534614016/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773162.153014:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773162.153019:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773162.153022:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773162.153029:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2158 00000100:00080000:1.0:1790773162.153037:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773162.153044:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773162.153058:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534614016:192.168.204.150@tcp:250:kworker.0 00000008:00100000:1.0:1790773162.153060:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773162.153090:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534614144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773162.153106:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773162.153110:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773162.153113:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773162.153119:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2158 00000100:00080000:1.0:1790773162.153124:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773162.153131:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773162.153135:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773162.153147:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773162.153164:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534614272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773162.153177:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773162.153183:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773162.153186:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773162.153192:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2158 00000100:00080000:1.0:1790773162.153198:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773162.153207:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773162.153209:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773162/real 1790773162] req@ffff9c4efac4fb80 x1877759534614016/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773162.153215:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773162.153246:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534614400/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773162.153249:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534614016/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773162.153317:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773162.153323:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773162.153326:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534614016:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773162.153338:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773162.153347:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773162.153354:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773162.153355:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534614144:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773162.153381:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2158 00000800:00000100:0.0:1790773162.153397:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773162.153404:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773162.153416:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773162/real 1790773162] req@ffff9c4efac4df80 x1877759534614144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773162.153418:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773162.153437:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773162.153456:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534614144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773162.153475:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534614528/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773162.153478:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773162.153492:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773162.153504:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534614144:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773162.153505:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773162.153520:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773162.153523:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773162.153530:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534614272:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773162.153535:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2158 00000100:00080000:1.0:1790773162.153545:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773162.153551:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773162.153557:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773162.153565:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773162.153575:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534614656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773162.153581:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773162/real 1790773162] req@ffff9c4efac4dc00 x1877759534614272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773162.153611:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773162.153621:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1790773162.153627:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534614272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773162.153627:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773162.153641:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2158 00000100:00080000:1.0:1790773162.153650:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773162.153655:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773162.153659:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773162.153675:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773162.153680:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773162.153683:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534614272:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773162.153691:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534614400:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773162.153698:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534614784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773162.153707:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773162.153715:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773162.153732:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773162.153735:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773162.153740:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773162/real 1790773162] req@ffff9c4efac4ed80 x1877759534614400/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773162.153742:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2158 00000100:00080000:1.0:1790773162.153755:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773162.153770:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534614400/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773162.153772:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773162.153786:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773162.153803:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790773162.153804:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773162.153810:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534614400:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773162.153831:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534614528:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773162.153844:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534614912/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773162.153852:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773162.153865:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773162/real 1790773162] req@ffff9c4efac4c700 x1877759534614528/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773162.153899:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773162.153918:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534614528/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773162.153960:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773162.153961:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773162.153967:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773162.153967:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773162.153972:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534614528:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773162.153980:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2158 00000100:00100000:0.0:1790773162.153987:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534614656:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773162.153991:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773162.154003:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773162.154025:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773162.154029:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773162.154038:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773162/real 1790773162] req@ffff9c4efac4f800 x1877759534614656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773162.154054:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534615040/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773162.154056:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534614656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773162.154161:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773162.154169:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773162.154175:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534614656:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773162.154189:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534614784:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773162.154214:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773162.154223:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773162/real 1790773162] req@ffff9c4efac4f480 x1877759534614784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773162.154232:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534614784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773162.154238:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773162.154241:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773162.154244:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534614784:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773162.154257:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534614912:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773162.154269:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773162.154276:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773162/real 1790773162] req@ffff9c4efac4e680 x1877759534614912/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773162.154283:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534614912/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773162.154291:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773162.154293:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773162.154295:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534614912:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773162.154299:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534615040:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773162.154310:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773162.154316:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773162/real 1790773162] req@ffff9c4efac4ea00 x1877759534615040/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773162.154324:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534615040/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773162.154330:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773162.154332:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773162.154334:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534615040:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773162.216734:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773162.216734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773162.216735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773162.216774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773162.280711:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773162.280717:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773162.280888:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773162.408714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773162.408728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773162.472740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773162.536708:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773162.600709:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773162.600734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773162.600753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773162.664714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773162.728739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773162.728784:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773162.728787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773162.856730:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773162.856731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773162.856741:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773162.856750:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773162.920718:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773162.920726:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773162.920733:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773162.920748:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773162.920770:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773162.920785:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773163.048715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773163.112739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773163.176718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773163.240729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773163.240733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773163.240735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773163.240810:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773163.304745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773163.304805:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773163.304911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773163.432726:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773163.432763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773163.496761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773163.560725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773163.624721:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773163.624726:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773163.624770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773163.688712:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773163.752724:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773163.752749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773163.752767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773163.880706:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773163.880722:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773163.880790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773163.880797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773163.944735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773163.944747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773163.944752:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773163.944765:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773163.944792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773163.944817:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773164.072724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773164.136711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773164.200715:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773164.264724:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773164.264733:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773164.264752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773164.264779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773164.328757:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773164.328775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773164.328885:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773164.456713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773164.456735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773164.520760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773164.584728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773164.648706:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773164.648728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773164.648769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773164.712731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773164.776739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773164.776742:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773164.776763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773164.904742:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773164.904748:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773164.904749:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773164.904760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773164.968725:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773164.968740:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773164.968762:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773164.968766:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773164.968769:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773164.968794:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773165.096703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773165.160726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773165.224722:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773165.288756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773165.288757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773165.288782:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773165.288787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773165.352762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773165.352766:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773165.352991:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773165.480731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773165.480732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773165.544763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773165.608715:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773165.672768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773165.672785:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773165.672807:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773165.736730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773165.800740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773165.800767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773165.800767:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773165.928735:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773165.928739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773165.928765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773165.928786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773165.992765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773165.992768:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773165.992773:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773165.992785:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773165.992801:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773165.992805:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773166.120712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773166.184723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773166.248719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773166.312763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773166.312763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773166.312781:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773166.312788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773166.376703:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773166.376762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773166.376901:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773166.504732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773166.504733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773166.568753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773166.632714:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773166.696734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773166.696739:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773166.696761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773166.760726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773166.824747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773166.824763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773166.824764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773166.952733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773166.952733:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773166.952775:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773166.952784:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773167.016747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773167.016761:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773167.016766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773167.016766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773167.016816:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773167.016826:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773167.144713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773167.208719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773167.272725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773167.272761:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773167.272767:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773167.272771:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773167.272779:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2163 00000100:00080000:3.0:1790773167.272796:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773167.272822:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773167.272857:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534615168/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773167.272880:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773167.272883:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773167.272884:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773167.272888:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2163 00000100:00080000:3.0:1790773167.272891:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773167.272895:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773167.272903:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773167.272914:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534615296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773167.272922:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534615168:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773167.272925:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773167.272932:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773167.272933:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773167.272943:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2163 00000100:00080000:3.0:1790773167.272947:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773167.272950:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773167.272954:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773167.272964:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534615424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773167.272971:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773167.272974:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773167.272975:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773167.272977:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2163 00000100:00080000:3.0:1790773167.272980:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773167.272983:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773167.272987:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773167.272994:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534615552/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773167.273001:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773167.273004:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773167.273005:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773167.273009:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773167.273013:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2163 00000100:00080000:3.0:1790773167.273016:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773167.273019:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773167.273022:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773167.273029:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534615680/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773167.273040:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773167.273043:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773167.273044:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773167.273048:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2163 00000100:00080000:3.0:1790773167.273050:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773167.273056:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773167.273057:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773167/real 1790773167] req@ffff9c4ef73c0000 x1877759534615168/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773167.273059:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773167.273070:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534615808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773167.273082:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773167.273085:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773167.273086:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773167.273091:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2163 00000100:00100000:0.0:1790773167.273093:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534615168/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773167.273094:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773167.273100:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773167.273105:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773167.273115:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534615936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773167.273118:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773167.273123:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773167.273127:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773167.273128:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773167.273129:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773167.273137:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2163 00000100:00100000:0.0:1790773167.273140:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534615168:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773167.273141:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773167.273148:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773167.273152:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773167.273159:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534616064/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773167.273165:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773167.273168:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773167.273169:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773167.273173:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534615296:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773167.273175:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2163 00000100:00080000:3.0:1790773167.273179:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773167.273185:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773167.273188:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773167.273195:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534616192/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773167.273212:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773167.273230:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773167/real 1790773167] req@ffff9c4ef73c1500 x1877759534615296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773167.273261:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534615296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773167.273280:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773167.273285:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773167.273288:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534615296:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773167.273296:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534615424:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773167.273319:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773167.273333:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773167/real 1790773167] req@ffff9c4ef73c0700 x1877759534615424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773167.273348:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534615424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773167.273357:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773167.273361:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773167.273365:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534615424:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773167.273372:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534615552:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773167.273392:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773167.273403:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773167/real 1790773167] req@ffff9c4ef73c2a00 x1877759534615552/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773167.273415:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534615552/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773167.273433:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773167.273437:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773167.273441:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534615552:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773167.273447:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534615680:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773167.273466:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773167.273476:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773167/real 1790773167] req@ffff9c4ef73c2d80 x1877759534615680/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773167.273488:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534615680/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773167.273498:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773167.273502:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773167.273506:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534615680:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773167.273513:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534615808:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773167.273534:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773167.273546:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773167/real 1790773167] req@ffff9c4ef73c3800 x1877759534615808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773167.273558:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534615808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773167.273575:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773167.273579:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773167.273584:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534615808:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773167.273591:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534615936:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773167.273613:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773167.273625:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773167/real 1790773167] req@ffff9c4ef73c0a80 x1877759534615936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773167.273638:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534615936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773167.273647:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773167.273672:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773167.273677:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534615936:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773167.273684:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534616064:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773167.273705:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773167.273717:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773167/real 1790773167] req@ffff9c4ef73c0e00 x1877759534616064/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773167.273730:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534616064/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773167.273742:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773167.273746:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773167.273750:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534616064:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773167.273775:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534616192:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773167.273798:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773167.273827:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773167/real 1790773167] req@ffff9c4ef73c2300 x1877759534616192/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773167.273841:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534616192/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773167.273852:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773167.273855:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773167.273860:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534616192:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773167.336732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773167.336732:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773167.336732:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773167.336739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773167.400720:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773167.400747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773167.400856:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773167.528716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773167.528720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773167.592742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773167.656735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773167.720731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773167.720731:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773167.720742:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773167.784731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773167.848724:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773167.848735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773167.848735:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773167.976736:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773167.976737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773167.976737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773167.976738:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773168.040716:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773168.040717:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773168.040741:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773168.040741:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773168.040762:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773168.040774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773168.168716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773168.232705:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773168.296706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773168.360725:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773168.360732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773168.360737:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773168.360737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773168.424725:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773168.424743:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773168.424834:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773168.552713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773168.552724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773168.616761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773168.680730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773168.744736:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773168.744736:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773168.744739:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773168.808714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773168.872755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773168.872755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773168.872764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773169.000711:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773169.000735:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773169.000737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773169.000737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773169.064741:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773169.064742:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773169.064790:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773169.064790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773169.064825:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773169.064826:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773169.192711:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773169.256725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773169.320717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773169.384715:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773169.384716:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773169.384722:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773169.384733:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773169.448711:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773169.448751:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773169.448854:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773169.576718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773169.576724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773169.640732:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773169.704743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773169.768722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773169.768722:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773169.768750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773169.832718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773169.896720:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773169.896722:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773169.896745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773170.024721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773170.024727:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773170.024793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773170.024801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773170.088719:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773170.088736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773170.088737:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773170.088760:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773170.088780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773170.088808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773170.216719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773170.280711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773170.344723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773170.408727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773170.408728:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773170.408729:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773170.408729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773170.472704:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773170.472731:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773170.472825:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773170.600703:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773170.600726:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773170.664722:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773170.728755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773170.792744:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773170.792744:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773170.792747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773170.856707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773170.920739:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773170.920740:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773170.920743:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773171.048714:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773171.048726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773171.048746:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773171.048747:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773171.112722:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773171.112742:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773171.112746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773171.112750:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773171.112771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773171.112779:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773171.240712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773171.304744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773171.368741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773171.432730:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773171.432752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773171.432756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773171.432776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773171.496747:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773171.496770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773171.496911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773171.624765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773171.624766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773171.688741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773171.752737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773171.816758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773171.816758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773171.816762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773171.880764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773171.944753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773171.944753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773171.944753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773172.072787:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773172.072854:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773172.072856:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773172.072861:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773172.136765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773172.136766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773172.136783:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773172.136783:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773172.136835:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773172.136838:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773172.264728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773172.328762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773172.392751:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773172.392754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773172.392763:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773172.392770:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773172.392785:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2168 00000100:00080000:3.0:1790773172.392801:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773172.392818:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773172.392869:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534616320/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773172.392897:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773172.392901:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773172.392903:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773172.392908:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2168 00000100:00080000:3.0:1790773172.392912:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773172.392919:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773172.392932:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773172.392939:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534616320:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773172.392947:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534616448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773172.392957:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773172.392961:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773172.392963:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773172.392967:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2168 00000100:00080000:3.0:1790773172.392970:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773172.392975:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773172.392980:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773172.392989:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534616576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773172.393004:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773172.393008:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773172.393010:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773172.393015:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2168 00000800:00000100:0.0:1790773172.393017:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773172.393018:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773172.393023:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773172.393027:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773172.393037:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534616704/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773172.393045:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773172.393049:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773172.393051:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773172.393055:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2168 00000100:00080000:3.0:1790773172.393058:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773172.393063:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773172.393066:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773172/real 1790773172] req@ffff9c4ef73c1c00 x1877759534616320/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773172.393068:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773172.393077:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534616832/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773172.393086:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534616320/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773172.393087:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773172.393091:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773172.393094:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773172.393098:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2168 00000100:00080000:0.0:1790773172.393100:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773172.393102:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773172.393105:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773172.393107:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773172.393110:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534616320:192.168.204.150@tcp:250:kworker.0 00000008:00100000:3.0:1790773172.393112:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773172.393121:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534616960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773172.393130:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773172.393134:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773172.393136:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773172.393137:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534616448:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773172.393141:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2168 00000100:00080000:3.0:1790773172.393145:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773172.393150:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773172.393155:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773172.393166:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534617088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773172.393171:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773172.393177:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773172.393181:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773172.393183:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773172.393186:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773172/real 1790773172] req@ffff9c4ef73c2300 x1877759534616448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773172.393188:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2168 00000100:00080000:3.0:1790773172.393192:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773172.393198:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773172.393200:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534616448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773172.393203:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773172.393213:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534617216/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773172.393220:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773172.393224:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773172.393228:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534616448:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773172.393229:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773172.393232:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1790773172.393235:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534616576:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773172.393235:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773172.393239:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2168 00000100:00080000:3.0:1790773172.393243:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773172.393248:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773172.393253:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790773172.393253:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773172.393264:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773172/real 1790773172] req@ffff9c4ef73c0e00 x1877759534616576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773172.393264:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534617344/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773172.393278:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534616576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773172.393288:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773172.393292:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773172.393296:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534616576:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773172.393304:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534616704:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773172.393326:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773172.393336:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773172/real 1790773172] req@ffff9c4ef73c0a80 x1877759534616704/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773172.393345:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534616704/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773172.393352:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773172.393355:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773172.393357:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534616704:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773172.393361:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534616832:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773172.393373:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773172.393379:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773172/real 1790773172] req@ffff9c4ef73c3800 x1877759534616832/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773172.393387:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534616832/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773172.393394:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773172.393396:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773172.393398:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534616832:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773172.393402:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534616960:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773172.393413:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773172.393419:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773172/real 1790773172] req@ffff9c4ef73c2d80 x1877759534616960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773172.393427:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534616960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773172.393439:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773172.393441:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773172.393443:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534616960:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773172.393458:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534617088:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773172.393470:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773172.393477:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773172/real 1790773172] req@ffff9c4ef73c2a00 x1877759534617088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773172.393485:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534617088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773172.393491:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773172.393493:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773172.393495:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534617088:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773172.393498:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534617216:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773172.393509:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773172.393515:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773172/real 1790773172] req@ffff9c4ef73c0700 x1877759534617216/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773172.393523:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534617216/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773172.393535:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773172.393538:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773172.393540:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534617216:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773172.393543:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534617344:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773172.393553:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773172.393559:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773172/real 1790773172] req@ffff9c4ef73c1500 x1877759534617344/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773172.393566:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534617344/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773172.393573:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773172.393575:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773172.393577:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534617344:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773172.456746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773172.456781:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773172.456800:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773172.456804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773172.520749:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773172.520769:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773172.520867:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773172.648720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773172.648735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773172.712750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773172.776762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773172.840742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773172.840789:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773172.840790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773172.904733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773172.968789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773172.968790:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773172.968793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773173.096733:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773173.096733:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773173.096762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773173.096763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773173.160765:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773173.160789:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773173.160790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773173.160794:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773173.160827:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773173.160828:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773173.288756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773173.352741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773173.416730:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773173.480768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773173.480768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773173.480769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773173.480769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773173.544766:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773173.544767:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773173.544782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773173.672733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773173.672737:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773173.736734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773173.800735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773173.864732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773173.864734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773173.864734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773173.928731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773173.992730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773173.992739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773173.992744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773174.120731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773174.120736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773174.120737:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773174.120738:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773174.184729:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773174.184731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773174.184732:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773174.184752:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773174.184764:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773174.184777:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773174.312760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773174.376747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773174.440722:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773174.504737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773174.504741:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773174.504743:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773174.504744:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773174.568757:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773174.568763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773174.568799:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773174.696733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773174.696774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773174.760757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773174.824727:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773174.888723:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773174.888726:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773174.888732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773174.952718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773175.016732:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773175.016732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773175.016757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773175.144731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773175.144735:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773175.144735:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773175.144740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773175.208716:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773175.208716:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773175.208733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773175.208750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773175.208764:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773175.208767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773175.336697:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773175.400728:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773175.464708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773175.528731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773175.528733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773175.528739:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773175.528740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773175.592725:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773175.592766:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773175.592802:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773175.720722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773175.720728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773175.784729:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773175.848737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773175.912742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773175.912743:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773175.912743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773175.976720:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773176.040719:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773176.040726:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773176.040734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773176.168740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773176.168740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773176.168740:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773176.168742:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773176.232710:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773176.232717:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773176.232733:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773176.232734:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773176.232755:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773176.232759:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773176.360718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773176.424725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773176.488712:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773176.552722:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773176.552723:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773176.552729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773176.552730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773176.616718:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773176.616733:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773176.616775:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773176.744717:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773176.744726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773176.808730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773176.872733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773176.936730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773176.936730:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773176.936730:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773177.000714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773177.064756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773177.064757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773177.064757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773177.192742:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773177.192742:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773177.192744:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773177.192759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773177.256725:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773177.256739:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773177.256743:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773177.256749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773177.256765:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773177.256779:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773177.384699:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773177.448719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773177.512713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773177.512714:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773177.512722:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773177.512726:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773177.512734:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2173 00000100:00080000:3.0:1790773177.512743:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773177.512760:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773177.512792:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534617472/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773177.512815:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773177.512818:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773177.512819:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773177.512822:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2173 00000100:00080000:3.0:1790773177.512825:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773177.512829:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773177.512837:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773177.512845:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534617472:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773177.512851:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534617600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773177.512859:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773177.512862:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773177.512863:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773177.512866:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2173 00000100:00080000:3.0:1790773177.512868:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773177.512871:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773177.512875:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773177.512884:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534617728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773177.512891:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773177.512894:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773177.512895:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773177.512898:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2173 00000100:00080000:3.0:1790773177.512902:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773177.512905:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773177.512909:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773177.512914:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773177.512923:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534617856/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773177.512930:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773177.512932:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773177.512934:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773177.512937:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2173 00000100:00080000:3.0:1790773177.512940:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773177.512944:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773177.512948:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773177/real 1790773177] req@ffff9c4ef73c0000 x1877759534617472/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773177.512950:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773177.512963:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534617984/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773177.512969:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534617472/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773177.512973:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773177.512980:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773177.512983:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773177.512989:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773177.512989:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2173 00000100:00080000:3.0:1790773177.512996:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773177.512998:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773177.513002:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773177.513005:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534617472:192.168.204.150@tcp:250:kworker.0 00000008:00100000:3.0:1790773177.513007:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773177.513017:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534618112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773177.513027:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534617600:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773177.513032:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773177.513035:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773177.513037:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773177.513042:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2173 00000100:00080000:3.0:1790773177.513045:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773177.513049:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773177.513051:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773177.513057:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773177.513063:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773177/real 1790773177] req@ffff9c4ef73c1500 x1877759534617600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773177.513067:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534618240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773177.513082:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534617600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773177.513083:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773177.513092:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773177.513095:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773177.513102:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2173 00000100:00080000:3.0:1790773177.513106:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773177.513110:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773177.513112:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773177.513114:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773177.513118:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773177.513123:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534617600:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773177.513126:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534618368/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773177.513132:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534617728:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773177.513140:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773177.513142:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773177.513144:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773177.513148:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2173 00000800:00000100:0.0:1790773177.513153:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773177.513154:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773177.513161:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773177.513165:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773177/real 1790773177] req@ffff9c4ef73c0700 x1877759534617728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773177.513166:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773177.513180:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534618496/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773177.513182:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534617728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773177.513211:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773177.513214:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773177.513217:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534617728:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773177.513223:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534617856:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773177.513246:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773177.513253:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773177/real 1790773177] req@ffff9c4ef73c2a00 x1877759534617856/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773177.513261:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534617856/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773177.513268:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773177.513271:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773177.513273:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534617856:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773177.513277:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534617984:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773177.513287:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773177.513293:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773177/real 1790773177] req@ffff9c4ef73c2d80 x1877759534617984/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773177.513301:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534617984/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773177.513307:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773177.513309:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773177.513312:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534617984:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773177.513336:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534618112:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773177.513357:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773177.513369:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773177/real 1790773177] req@ffff9c4ef73c3800 x1877759534618112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773177.513383:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534618112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773177.513397:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773177.513408:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773177.513412:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534618112:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773177.513419:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534618240:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773177.513442:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773177.513454:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773177/real 1790773177] req@ffff9c4ef73c0a80 x1877759534618240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773177.513464:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534618240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773177.513470:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773177.513472:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773177.513475:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534618240:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773177.513479:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534618368:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773177.513493:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773177.513501:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773177/real 1790773177] req@ffff9c4ef73c0e00 x1877759534618368/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773177.513509:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534618368/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773177.513516:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773177.513518:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773177.513520:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534618368:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773177.513523:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534618496:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773177.513534:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773177.513540:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773177/real 1790773177] req@ffff9c4ef73c2300 x1877759534618496/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773177.513547:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534618496/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773177.513554:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773177.513556:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773177.513558:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534618496:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773177.576752:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773177.576752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773177.576753:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773177.576753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773177.640740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773177.640767:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773177.640789:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773177.768716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773177.768716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773177.832737:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773177.896722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773177.960726:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773177.960727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773177.960728:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773178.024703:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773178.088711:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773178.088722:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773178.088725:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773178.216752:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773178.216753:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773178.216753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773178.216753:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773178.280719:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773178.280741:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773178.280749:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773178.280761:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773178.280762:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773178.280775:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773178.408733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773178.472731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773178.536717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773178.600713:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773178.600723:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773178.600729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773178.600731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773178.664736:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773178.664742:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773178.664846:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773178.792718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773178.792727:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773178.856740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773178.920747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773178.984719:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773178.984719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773178.984722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773179.048722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773179.112735:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773179.112737:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773179.112738:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773179.240697:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773179.240723:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773179.240725:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773179.240726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773179.304729:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773179.304735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773179.304738:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773179.304747:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773179.304765:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773179.304773:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773179.432716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773179.496739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773179.560711:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773179.624737:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773179.624737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773179.624740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773179.624750:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773179.688724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773179.688734:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773179.688812:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773179.816748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773179.816777:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773179.880724:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773179.944845:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773180.008819:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773180.008841:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773180.008845:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773180.072761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773180.136761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773180.136784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773180.136817:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773180.264766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773180.264783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773180.264804:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773180.264805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773180.328730:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773180.328732:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773180.328734:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773180.328750:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773180.328770:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773180.328784:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773180.456722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773180.520722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773180.584761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773180.648762:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773180.648786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773180.648814:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773180.648828:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773180.712803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773180.712817:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773180.712946:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773180.840752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773180.840784:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773180.904740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773180.968721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773181.032755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773181.032756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773181.032772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773181.096712:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773181.160711:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773181.160720:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773181.160721:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773181.288771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773181.288786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773181.288794:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773181.288795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773181.352715:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773181.352745:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773181.352750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773181.352751:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773181.352762:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773181.352787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773181.480730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773181.544765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773181.608763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773181.672712:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773181.672732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773181.672737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773181.672754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773181.736768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773181.736822:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773181.736896:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773181.864718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773181.864722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773181.928755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773181.992736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773182.056713:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773182.056722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773182.056722:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773182.120701:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773182.184712:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773182.184731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773182.184731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773182.312824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773182.312824:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773182.312828:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773182.312830:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773182.376743:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773182.376808:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773182.376808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773182.376818:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773182.376838:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773182.376896:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773182.504770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773182.568836:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773182.632744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773182.632824:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773182.632856:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773182.632862:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773182.632876:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2178 00000100:00080000:3.0:1790773182.632891:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773182.632927:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773182.632995:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534618624/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773182.633041:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773182.633047:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773182.633050:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773182.633056:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2178 00000100:00080000:3.0:1790773182.633063:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773182.633073:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773182.633091:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773182.633118:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534618624:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773182.633119:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534618752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773182.633168:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773182.633175:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773182.633180:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773182.633188:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2178 00000100:00080000:3.0:1790773182.633194:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773182.633220:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773182.633226:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773182.633239:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534618880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773182.633249:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773182.633259:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1790773182.633261:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773182.633286:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773182.633298:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2178 00000100:00080000:3.0:1790773182.633305:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773182.633315:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773182.633324:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773182.633341:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534619008/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773182.633355:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773182.633371:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773182/real 1790773182] req@ffff9c4ef73c1c00 x1877759534618624/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773182.633387:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773182.633394:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773182.633407:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2178 00000100:00080000:3.0:1790773182.633419:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773182.633450:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773182.633459:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773182.633470:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534618624/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773182.633485:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534619136/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773182.633536:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773182.633550:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773182.633552:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773182.633563:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1790773182.633565:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534618624:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773182.633575:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773182.633611:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2178 00000100:00080000:3.0:1790773182.633622:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773182.633632:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773182.633643:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534618752:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790773182.633644:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773182.633727:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534619264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773182.633740:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773182.633741:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773182.633755:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773182.633758:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773182.633793:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2178 00000100:00100000:0.0:1790773182.633795:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773182/real 1790773182] req@ffff9c4ef73c2300 x1877759534618752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773182.633809:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773182.633834:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773182.633853:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534618752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773182.633862:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773182.633902:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534619392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773182.633903:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773182.633918:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773182.633941:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534618752:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773182.633974:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773182.633979:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534618880:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773182.633992:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773182.633999:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773182.634016:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2178 00000100:00080000:3.0:1790773182.634029:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773182.634041:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773182.634048:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790773182.634056:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773182.634074:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773182/real 1790773182] req@ffff9c4ef73c0e00 x1877759534618880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773182.634084:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534619520/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773182.634170:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773182.634180:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1790773182.634181:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534618880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773182.634187:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773182.634211:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773182.634218:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2178 00000100:00080000:0.0:1790773182.634227:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773182.634231:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773182.634239:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534618880:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773182.634246:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773182.634279:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534619008:192.168.204.150@tcp:8:kworker.0 00000008:00100000:3.0:1790773182.634282:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773182.634317:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534619648/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773182.634350:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773182.634397:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773182/real 1790773182] req@ffff9c4ef73c0a80 x1877759534619008/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773182.634425:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534619008/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773182.634452:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773182.634457:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773182.634462:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534619008:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773182.634473:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534619136:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773182.634530:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773182.634551:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773182/real 1790773182] req@ffff9c4ef73c3800 x1877759534619136/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773182.634577:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534619136/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773182.634589:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773182.634594:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773182.634598:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534619136:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773182.634627:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534619264:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773182.634688:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773182.634714:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773182/real 1790773182] req@ffff9c4ef73c2d80 x1877759534619264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773182.634731:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534619264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773182.634750:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773182.634755:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773182.634760:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534619264:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773182.634780:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534619392:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773182.634803:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773182.634815:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773182/real 1790773182] req@ffff9c4ef73c2a00 x1877759534619392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773182.634830:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534619392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773182.634861:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773182.634866:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773182.634870:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534619392:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773182.634878:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534619520:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773182.634898:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773182.634909:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773182/real 1790773182] req@ffff9c4ef73c0700 x1877759534619520/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773182.634924:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534619520/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773182.634942:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773182.634946:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773182.634950:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534619520:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773182.634958:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534619648:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773182.634978:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773182.634990:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773182/real 1790773182] req@ffff9c4ef73c1500 x1877759534619648/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773182.635005:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534619648/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773182.635017:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773182.635021:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773182.635027:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534619648:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773182.696786:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773182.696791:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773182.696829:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773182.696836:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773182.760774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773182.760874:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773182.760954:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773182.888796:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773182.888804:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773182.952783:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773183.016787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773183.080805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773183.080860:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773183.080882:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773183.144752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773183.208773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773183.208780:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773183.208823:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773183.336789:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773183.336826:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773183.336830:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773183.336833:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773183.400790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773183.400794:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773183.400820:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773183.400835:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773183.400871:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773183.400887:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773183.528734:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773183.592735:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773183.656775:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773183.720761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773183.720802:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773183.720807:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773183.720808:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773183.784803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773183.784820:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773183.784973:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773183.912772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773183.912796:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773183.976808:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773184.040808:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773184.104768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773184.104777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773184.104782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773184.168766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773184.232755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773184.232769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773184.232780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773184.360763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773184.360774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773184.360800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773184.360809:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773184.424766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773184.424793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773184.424801:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773184.424820:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773184.424851:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773184.424865:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773184.552744:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773184.616726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773184.680711:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773184.744787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773184.744795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773184.744795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773184.744839:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773184.808798:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773184.808882:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773184.808964:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773184.936758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773184.936786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773185.000790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773185.064815:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773185.128742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773185.128764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773185.128831:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773185.192707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773185.256786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773185.256798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773185.256818:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773185.384770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773185.384794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773185.384801:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773185.384816:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773185.448779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773185.448784:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773185.448831:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773185.448831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773185.448874:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773185.448878:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773185.576756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773185.640762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773185.704789:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773185.768781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773185.768796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773185.768829:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773185.768830:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773185.832802:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773185.832830:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773185.833016:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773185.960727:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773185.960773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773186.024764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773186.088778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773186.152765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773186.152784:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773186.152795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773186.216768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773186.280762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773186.280781:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773186.280790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773186.408746:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773186.408763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773186.408768:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773186.408786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773186.472773:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773186.472788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773186.472812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773186.472812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773186.472834:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773186.472847:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773186.600760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773186.664758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773186.728771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773186.792768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773186.792779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773186.792792:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773186.792792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773186.856779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773186.856812:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773186.857001:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773186.984781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773186.984796:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773187.048811:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773187.112840:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773187.176789:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773187.176798:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773187.176801:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773187.240756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773187.304779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773187.304842:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773187.304846:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773187.432868:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773187.432869:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773187.432872:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773187.432880:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773187.496733:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773187.496738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773187.496740:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773187.496780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773187.496783:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773187.496808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773187.624742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773187.688784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773187.752770:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773187.752834:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773187.752847:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773187.752855:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773187.752869:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2183 00000100:00080000:2.0:1790773187.752884:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773187.752899:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773187.752950:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534619776/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773187.752993:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773187.752999:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773187.753001:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773187.753008:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2183 00000100:00080000:2.0:1790773187.753013:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773187.753020:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773187.753032:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773187.753052:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534619904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773187.753080:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534619776:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773187.753080:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773187.753109:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773187.753114:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773187.753122:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2183 00000100:00080000:2.0:1790773187.753132:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773187.753146:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773187.753158:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773187.753194:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534620032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773187.753217:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773187.753225:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773187.753230:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773187.753242:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2183 00000100:00080000:2.0:1790773187.753250:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773187.753261:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773187.753271:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773187.753292:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534619904:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773187.753298:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534620160/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773187.753323:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773187.753329:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773187.753334:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773187.753342:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2183 00000100:00080000:2.0:1790773187.753355:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773187.753363:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534620032:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773187.753374:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773187.753388:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773187.753450:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534620160:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773187.753455:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534620288/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773187.753490:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773187.753499:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773187.753506:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773187.753516:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534620288:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773187.753526:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2183 00000100:00080000:2.0:1790773187.753544:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773187.753562:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773187.753579:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773187.753615:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534620416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773187.753709:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773187.753716:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773187.753719:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773187.753728:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534620416:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773187.753732:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2183 00000100:00080000:2.0:1790773187.753744:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773187.753756:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773187.753769:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773187.753799:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534620544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773187.753824:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773187.753830:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773187.753833:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773187.753845:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534620544:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773187.753847:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2183 00000100:00080000:2.0:1790773187.753859:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773187.753879:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773187.753891:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773187.753918:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534620672/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773187.753946:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773187.753954:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773187.753960:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773187.753970:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2183 00000100:00080000:2.0:1790773187.753980:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773187.753991:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534620672:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773187.754001:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773187.754016:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773187.754070:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534620800/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773187.754116:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534620800:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773187.816793:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773187.816797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773187.816848:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773187.816872:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773187.880785:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773187.880860:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773187.881071:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773188.008707:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773188.008708:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773188.072727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773188.136764:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773188.200739:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773188.200748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773188.200774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773188.264747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773188.328773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773188.328778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773188.328782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773188.456771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773188.456785:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773188.456805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773188.456813:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773188.520763:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773188.520782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773188.520814:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773188.520821:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773188.520835:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773188.520846:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773188.648800:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773188.712747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773188.776765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773188.840717:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773188.840734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773188.840734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773188.840737:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773188.904731:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773188.904743:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773188.904921:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773189.032744:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773189.032760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773189.096782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773189.160730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773189.224747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773189.224747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773189.224748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773189.288746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773189.352779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773189.352783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773189.352790:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773189.480743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773189.480774:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773189.480778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773189.480778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773189.544758:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773189.544769:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773189.544786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773189.544793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773189.544822:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773189.544825:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773189.672763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773189.736741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773189.800762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773189.864771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773189.864772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773189.864781:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773189.864784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773189.928760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773189.928803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773189.928968:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773190.056727:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773190.056774:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773190.120747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773190.184752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773190.248767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773190.248803:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773190.248815:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773190.312771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773190.376754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773190.376796:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773190.376796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773190.504743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773190.504810:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773190.504824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773190.504834:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773190.568799:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773190.568812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773190.568834:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773190.568843:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773190.568882:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773190.568902:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773190.696780:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773190.760736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773190.824747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:1.0:1790773190.825049:0:2386:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:1.0:1790773190.825109:0:2386:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1790773190.825137:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773190.825218:0:2386:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:1.0:1790773190.825237:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773190.825252:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773190.825267:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773190.825279:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773190.825297:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773187/real 1790773190] req@ffff9c4efbcc2a00 x1877759534620800/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790773190.825297:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773190.825318:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773190.825337:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773190.825401:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534620800/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773190.825434:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773190.825446:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773190.825457:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534620800:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773190.825474:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773187/real 1790773190] req@ffff9c4efbcc1c00 x1877759534620672/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773190.825504:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534620672/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773190.825525:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773190.825532:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773190.825541:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534620672:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773190.825555:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773187/real 1790773190] req@ffff9c4efbcc1f80 x1877759534620544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773190.825580:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534620544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773190.825612:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773190.825621:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773190.825629:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534620544:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773190.825644:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773187/real 1790773190] req@ffff9c4efbcc1500 x1877759534620416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773190.825722:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534620416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773190.825756:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773190.825765:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773190.825773:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534620416:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773190.825789:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773187/real 1790773190] req@ffff9c4efbcc1880 x1877759534620288/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773190.825816:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534620288/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773190.825839:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773190.825846:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773190.825854:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534620288:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773190.825869:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773187/real 1790773190] req@ffff9c4efbcc2300 x1877759534620160/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773190.825895:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534620160/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773190.825919:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773190.825927:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773190.825935:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534620160:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773190.825953:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773187/real 1790773190] req@ffff9c4efbcc0a80 x1877759534619904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773190.825984:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534619904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773190.826016:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773190.826025:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773190.826033:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534619904:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773190.826049:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773187/real 1790773190] req@ffff9c4efbcc2680 x1877759534620032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773190.826078:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534620032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773190.826097:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773190.826105:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773190.826112:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534620032:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773190.826129:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773187/real 1790773190] req@ffff9c4efbcc0700 x1877759534619776/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773190.826170:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534619776/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773190.826196:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773190.826204:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773190.826214:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534619776:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773190.888786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773190.888786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773190.888803:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773190.888810:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773190.952861:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773190.952924:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773190.953003:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773191.080736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773191.080792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773191.144768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773191.208838:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773191.272737:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773191.272741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773191.272767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773191.336783:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773191.400828:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773191.400831:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773191.400847:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773191.528732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773191.528786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773191.528796:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773191.528803:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773191.592788:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773191.592828:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773191.592852:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773191.592868:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773191.592870:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773191.592891:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773191.720755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773191.784788:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773191.848750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773191.912776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773191.912803:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773191.912821:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773191.912860:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773191.976750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773191.976800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773191.977006:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773192.104819:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773192.104851:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773192.168769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773192.232818:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773192.296751:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773192.296767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773192.296793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773192.360750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773192.424760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773192.424766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773192.424772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773192.552727:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773192.552762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773192.552765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773192.552792:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773192.616787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773192.616810:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773192.616812:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773192.616852:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773192.616853:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773192.616870:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773192.744766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773192.808773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773192.872784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773192.872917:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773192.872934:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773192.872941:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773192.872959:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2188 00000100:00080000:1.0:1790773192.872976:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773192.872997:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773192.873057:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534620928/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773192.873111:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773192.873117:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773192.873120:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773192.873127:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2188 00000100:00080000:1.0:1790773192.873132:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773192.873141:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773192.873158:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773192.873193:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534620928:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773192.873195:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534621056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773192.873232:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773192.873240:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773192.873259:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773192.873270:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2188 00000100:00080000:1.0:1790773192.873292:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773192.873332:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773192.873344:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773192.873363:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773192.873398:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534621184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773192.873421:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773192.873434:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1790773192.873435:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773192/real 1790773192] req@ffff9c4efac4e300 x1877759534620928/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773192.873443:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773192.873477:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2188 00000100:00080000:1.0:1790773192.873491:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773192.873498:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534620928/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773192.873517:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773192.873541:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00080000:0.0:1790773192.873544:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773192.873564:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773192.873581:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534620928:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773192.873588:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534621312/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773192.873617:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773192.873624:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773192.873630:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773192.873704:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2188 00000100:00080000:1.0:1790773192.873716:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773192.873725:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534621056:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773192.873732:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773192.873754:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773192.873809:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534621440/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773192.873829:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773192.873858:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773192.873869:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773192.873875:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773192.873886:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773192/real 1790773192] req@ffff9c4efac4ea00 x1877759534621056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773192.873895:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2188 00000100:00080000:1.0:1790773192.873913:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773192.873938:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773192.873955:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773192.873957:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534621056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773192.873996:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534621568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773192.874006:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773192.874033:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773192.874040:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773192.874052:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534621056:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773192.874055:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773192.874066:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773192.874087:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2188 00000100:00100000:0.0:1790773192.874094:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534621184:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773192.874102:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773192.874122:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773192.874136:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773192.874173:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773192.874175:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534621696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773192.874206:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773192.874218:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773192/real 1790773192] req@ffff9c4efac4e680 x1877759534621184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773192.874221:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773192.874230:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773192.874258:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2188 00000100:00080000:1.0:1790773192.874276:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773192.874286:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534621184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773192.874299:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773192.874339:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773192.874359:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773192.874375:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773192.874382:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534621824/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773192.874394:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534621184:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773192.874435:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534621312:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773192.874437:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773192.874459:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773192.874464:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773192.874480:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2188 00000800:00000100:0.0:1790773192.874514:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773192.874523:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773192.874550:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773192.874564:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773192/real 1790773192] req@ffff9c4efac4f480 x1877759534621312/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773192.874572:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773192.874609:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534621952/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773192.874628:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534621312/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773192.874749:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773192.874766:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773192.874805:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534621312:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773192.874860:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534621440:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773192.874912:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773192.874951:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773192/real 1790773192] req@ffff9c4efac4f800 x1877759534621440/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773192.874978:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534621440/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773192.874998:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773192.875005:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773192.875013:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534621440:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773192.875027:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534621568:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773192.875072:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773192.875097:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773192/real 1790773192] req@ffff9c4efac4c700 x1877759534621568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773192.875125:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534621568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773192.875155:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773192.875163:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773192.875170:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534621568:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773192.875184:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534621696:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773192.875228:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773192.875252:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773192/real 1790773192] req@ffff9c4efac4ed80 x1877759534621696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773192.875277:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534621696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773192.875327:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773192.875336:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773192.875344:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534621696:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773192.875358:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534621824:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773192.875402:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773192.875426:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773192/real 1790773192] req@ffff9c4efac4dc00 x1877759534621824/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773192.875449:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534621824/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773192.875482:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773192.875490:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773192.875497:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534621824:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773192.875510:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534621952:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773192.875551:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773192.875574:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773192/real 1790773192] req@ffff9c4efac4df80 x1877759534621952/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773192.875595:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534621952/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773192.875608:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773192.875613:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773192.875618:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534621952:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773192.936703:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773192.936753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773192.936819:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773192.936836:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773193.000728:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773193.000817:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773193.000981:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773193.128734:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773193.128739:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773193.192757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773193.256739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773193.320764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773193.320769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773193.320799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773193.384777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773193.448789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773193.448794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773193.448802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773193.576775:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773193.576779:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773193.576781:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773193.576801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773193.640792:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773193.640792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773193.640839:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773193.640861:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773193.640866:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773193.640873:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773193.768745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773193.832763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773193.896710:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773193.960732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773193.960753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773193.960759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773193.960759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773194.024777:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773194.024790:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773194.024924:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773194.152778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773194.152791:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773194.216754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773194.280732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773194.344726:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773194.344734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773194.344735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773194.408746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773194.472735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773194.472743:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773194.472770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773194.600764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773194.600769:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773194.600780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773194.600806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773194.664775:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773194.664787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773194.664795:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773194.664799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773194.664843:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773194.664844:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773194.792735:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773194.856737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773194.920709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773194.984786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773194.984801:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773194.984842:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773194.984846:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773195.048784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773195.048838:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773195.048869:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773195.176720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773195.176727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773195.240788:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773195.304825:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773195.368736:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773195.368788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773195.368789:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773195.432715:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773195.496793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773195.496801:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773195.496804:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773195.624737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773195.624744:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773195.624762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773195.624773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773195.688774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773195.688785:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773195.688812:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773195.688833:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773195.688863:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773195.688883:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773195.816700:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773195.880765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773195.944721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773196.008714:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773196.008716:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773196.008719:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773196.008723:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773196.072711:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773196.072734:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773196.072761:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773196.200713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773196.200714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773196.264730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773196.328711:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773196.392721:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773196.392722:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773196.392724:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773196.456709:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773196.520731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773196.520732:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773196.520732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773196.648770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773196.648776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773196.648783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773196.648797:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773196.712758:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773196.712787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773196.712801:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773196.712851:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773196.712859:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773196.712879:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773196.840745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773196.904742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773196.968763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773197.032826:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773197.032852:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773197.032869:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773197.032899:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773197.096756:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773197.096853:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773197.096976:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773197.224761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773197.224789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773197.288787:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773197.352808:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773197.416766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773197.416783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773197.416821:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773197.480769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773197.544771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773197.544783:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773197.544826:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773197.672858:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773197.672859:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773197.672859:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773197.672859:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773197.736820:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773197.736828:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773197.736837:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773197.736890:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773197.736950:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773197.736955:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773197.864733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773197.928769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773197.992769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773197.992783:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773197.992794:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773197.992800:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773197.992815:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2193 00000100:00080000:1.0:1790773197.992829:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773197.992848:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773197.992911:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534622080/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773197.992979:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773197.992985:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773197.992988:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773197.992995:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2193 00000100:00080000:1.0:1790773197.993002:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773197.993012:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773197.993027:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773197.993038:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534622080:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773197.993054:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534622208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773197.993073:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773197.993078:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773197.993082:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773197.993088:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2193 00000100:00080000:1.0:1790773197.993115:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773197.993122:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773197.993133:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773197.993136:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773197.993158:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534622336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773197.993173:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773197.993179:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773197.993184:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773197.993194:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773197/real 1790773197] req@ffff9c4efac4fb80 x1877759534622080/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773197.993200:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2193 00000100:00080000:1.0:1790773197.993238:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773197.993256:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773197.993262:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534622080/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773197.993274:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00080000:0.0:1790773197.993281:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773197.993290:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773197.993296:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534622080:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773197.993311:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534622464/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773197.993332:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773197.993335:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534622208:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773197.993344:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773197.993351:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773197.993359:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2193 00000100:00080000:1.0:1790773197.993367:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773197.993393:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773197.993396:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773197.993409:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773197.993433:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773197/real 1790773197] req@ffff9c4efac4df80 x1877759534622208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773197.993438:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534622592/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773197.993471:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534622208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773197.993524:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773197.993525:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773197.993539:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773197.993540:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773197.993548:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773197.993552:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534622208:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773197.993560:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2193 00000100:00080000:1.0:1790773197.993570:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773197.993572:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534622336:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773197.993585:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773197.993596:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773197.993616:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773197.993624:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534622720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773197.993640:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773197/real 1790773197] req@ffff9c4efac4dc00 x1877759534622336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773197.993641:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773197.993705:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773197.993709:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773197.993712:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534622336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773197.993720:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2193 00000100:00080000:1.0:1790773197.993732:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773197.993741:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773197.993743:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773197.993754:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790773197.993756:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773197.993763:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534622336:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773197.993784:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534622848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773197.993797:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534622464:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773197.993807:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773197.993815:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773197.993819:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773197.993827:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2193 00000800:00000100:0.0:1790773197.993837:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773197.993841:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773197.993860:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773197.993861:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773197/real 1790773197] req@ffff9c4efac4ed80 x1877759534622464/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773197.993881:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773197.993888:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534622464/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773197.993905:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773197.993916:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773197.993920:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534622976/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773197.993923:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534622464:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773197.993944:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534622592:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773197.993968:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773197.993974:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773197.993978:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773197.993985:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773197.993993:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2193 00000100:00080000:1.0:1790773197.994003:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773197.994009:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773197/real 1790773197] req@ffff9c4efac4c700 x1877759534622592/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773197.994018:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773197.994037:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773197.994040:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534622592/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773197.994062:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534623104/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773197.994071:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773197.994085:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773197.994094:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534622592:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773197.994104:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534622720:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773197.994131:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773197.994144:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773197/real 1790773197] req@ffff9c4efac4f800 x1877759534622720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773197.994159:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534622720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773197.994178:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773197.994182:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773197.994187:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534622720:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773197.994238:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534622848:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773197.994268:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773197.994283:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773197/real 1790773197] req@ffff9c4efac4f480 x1877759534622848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773197.994299:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534622848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773197.994310:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773197.994314:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773197.994321:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534622848:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773197.994329:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534622976:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773197.994348:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773197.994362:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773197/real 1790773197] req@ffff9c4efac4e680 x1877759534622976/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773197.994387:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534622976/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773197.994409:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773197.994416:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773197.994424:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534622976:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773197.994437:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534623104:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773197.994477:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773197.994494:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773197/real 1790773197] req@ffff9c4efac4ea00 x1877759534623104/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773197.994510:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534623104/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773197.994523:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773197.994528:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773197.994533:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534623104:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773198.056769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773198.056799:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773198.056809:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773198.056809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773198.120788:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773198.120808:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773198.120906:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773198.248742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773198.248745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773198.312728:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773198.376776:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773198.440761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773198.440791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773198.440793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773198.504800:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773198.568755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773198.568774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773198.568776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773198.696782:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773198.696790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773198.696800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773198.696827:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773198.760752:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773198.760753:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773198.760757:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773198.760789:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773198.760830:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773198.760863:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773198.888750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773198.952754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773199.016745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773199.080837:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773199.080843:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773199.080844:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773199.080846:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773199.144799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773199.144874:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773199.144926:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773199.272769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773199.272798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773199.336740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773199.400730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773199.464835:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773199.464843:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773199.464865:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773199.528710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773199.592708:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773199.592714:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773199.592735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773199.720777:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773199.720865:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773199.720868:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773199.720875:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773199.784746:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773199.784757:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773199.784779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773199.784816:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773199.784831:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773199.784833:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773199.912763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773199.976757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773200.040771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773200.104742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773200.104796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773200.104797:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773200.104797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773200.168717:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773200.168763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773200.168844:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773200.296727:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773200.296732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773200.360811:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773200.424735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773200.488730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773200.488730:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773200.488745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773200.552736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773200.616718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773200.616722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773200.616750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773200.744737:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773200.744747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773200.744770:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773200.744804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773200.808708:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773200.808726:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773200.808728:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773200.808762:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773200.808824:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773200.808855:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773200.936786:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773201.000767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773201.064703:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773201.128729:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773201.128729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773201.128731:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773201.128733:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773201.192741:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773201.192765:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773201.192847:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773201.320735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773201.320771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773201.384753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773201.448833:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773201.512750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773201.512787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773201.512851:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773201.576761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773201.640737:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773201.640740:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773201.640779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773201.768794:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773201.768848:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773201.768880:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773201.768880:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773201.832750:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773201.832812:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773201.832812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773201.832812:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773201.832835:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773201.832881:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773201.960750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773202.024760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773202.088719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773202.152755:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773202.152778:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773202.152783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773202.152799:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773202.216767:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773202.216899:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773202.216955:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773202.344733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773202.344768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773202.408762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773202.472815:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773202.536762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773202.536784:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773202.536794:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773202.600804:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773202.664810:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773202.664855:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773202.664891:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773202.792787:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773202.792798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773202.792811:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773202.792861:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773202.856716:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773202.856769:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773202.856794:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773202.856801:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773202.856802:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773202.856832:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773202.984751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773203.048753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773203.112770:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773203.112782:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773203.112788:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773203.112794:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773203.112803:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2199 00000100:00080000:1.0:1790773203.112820:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773203.112837:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773203.112884:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534623232/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773203.112923:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773203.112929:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773203.112932:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773203.112937:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2199 00000100:00080000:1.0:1790773203.112943:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773203.112951:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773203.112963:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773203.112983:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534623232:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773203.112987:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534623360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773203.113010:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773203.113016:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773203.113019:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773203.113026:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2199 00000100:00080000:1.0:1790773203.113032:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773203.113039:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773203.113047:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773203.113061:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534623488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773203.113075:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773203.113083:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773203.113086:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773203.113092:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773203.113101:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2199 00000100:00080000:1.0:1790773203.113139:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773203.113151:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773203.113169:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773203.113195:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773203/real 1790773203] req@ffff9c4efac4e300 x1877759534623232/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773203.113197:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534623616/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773203.113264:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773203.113268:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534623232/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773203.113271:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773203.113278:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773203.113294:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2199 00000100:00080000:1.0:1790773203.113304:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773203.113309:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773203.113312:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773203.113327:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:0.0:1790773203.113329:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773203.113344:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534623232:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773203.113347:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534623744/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773203.113379:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773203.113384:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773203.113387:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773203.113393:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2199 00000100:00080000:1.0:1790773203.113397:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773203.113404:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773203.113412:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534623360:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790773203.113414:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773203.113438:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534623872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773203.113454:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773203.113460:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:0.0:1790773203.113468:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773203.113469:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773203.113481:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2199 00000100:00080000:1.0:1790773203.113488:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773203.113498:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773203/real 1790773203] req@ffff9c4efac4ea00 x1877759534623360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773203.113499:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773203.113514:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773203.113540:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534623360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773203.113547:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534624000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773203.113578:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773203.113583:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773203.113585:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773203.113594:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773203.113599:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773203.113608:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2199 00000100:00100000:0.0:1790773203.113612:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534623360:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773203.113616:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773203.113631:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773203.113633:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534623488:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790773203.113646:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:0.0:1790773203.113710:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773203.113718:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534624128/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773203.113732:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773203/real 1790773203] req@ffff9c4efac4e680 x1877759534623488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773203.113739:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773203.113756:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773203.113763:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773203.113772:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534623488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773203.113773:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2199 00000100:00080000:1.0:1790773203.113787:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773203.113801:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773203.113802:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773203.113816:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790773203.113817:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773203.113828:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534623488:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773203.113841:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534624256/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773203.113843:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534623616:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773203.113885:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773203.113904:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773203/real 1790773203] req@ffff9c4efac4f480 x1877759534623616/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773203.113920:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534623616/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773203.113932:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773203.113936:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773203.113940:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534623616:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773203.113948:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534623744:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773203.113967:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773203.113980:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773203/real 1790773203] req@ffff9c4efac4f800 x1877759534623744/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773203.113995:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534623744/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773203.114006:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773203.114010:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773203.114014:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534623744:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773203.114035:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534623872:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773203.114057:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773203.114069:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773203/real 1790773203] req@ffff9c4efac4c700 x1877759534623872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773203.114084:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534623872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773203.114128:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773203.114134:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773203.114139:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534623872:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773203.114158:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534624000:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773203.114186:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773203.114202:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773203/real 1790773203] req@ffff9c4efac4ed80 x1877759534624000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773203.114217:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534624000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773203.114229:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773203.114233:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773203.114238:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534624000:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773203.114248:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534624128:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773203.114280:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773203.114301:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773203/real 1790773203] req@ffff9c4efac4dc00 x1877759534624128/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773203.114328:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534624128/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773203.114350:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773203.114358:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773203.114364:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534624128:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773203.114373:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534624256:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773203.114412:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773203.114428:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773203/real 1790773203] req@ffff9c4efac4df80 x1877759534624256/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773203.114443:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534624256/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773203.114466:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773203.114470:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773203.114476:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534624256:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773203.176786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773203.176794:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773203.176813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773203.176825:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773203.240779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773203.240802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773203.240888:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773203.368775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773203.368808:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773203.432740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773203.496808:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773203.560759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773203.560767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773203.560767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773203.624704:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773203.688759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773203.688759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773203.688792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773203.816727:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773203.816729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773203.816742:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773203.816759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773203.880762:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773203.880796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773203.880797:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773203.880797:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773203.880871:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773203.880872:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773204.008730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773204.072757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773204.136759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773204.200758:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773204.200822:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773204.200822:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773204.200838:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773204.264845:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773204.264850:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773204.264895:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773204.392816:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773204.392828:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773204.456767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773204.520804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773204.584761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773204.584775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773204.584781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773204.648773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773204.712771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773204.712829:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773204.712832:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773204.840732:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773204.840733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773204.840819:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773204.840819:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773204.904787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773204.904799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773204.904832:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773204.904849:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773204.904893:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773204.904898:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773205.032789:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773205.096746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773205.161230:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773205.224786:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773205.224786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773205.224791:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773205.224817:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773205.288816:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773205.288816:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773205.288870:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773205.416797:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773205.416797:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773205.480818:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773205.544784:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773205.608759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773205.608771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773205.608827:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773205.672777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773205.736799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773205.736806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773205.736813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773205.864791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773205.864791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773205.864806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773205.864806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773205.928783:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773205.928802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773205.928817:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773205.928817:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773205.928858:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773205.928870:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773206.056724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773206.120780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773206.184818:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773206.248737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773206.248788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773206.248810:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773206.248814:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773206.312807:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773206.312812:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773206.312865:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773206.440740:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773206.440740:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773206.504801:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773206.568728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773206.632735:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773206.632764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773206.632820:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773206.696757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773206.760819:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773206.760821:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773206.760823:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773206.888786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773206.888808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773206.888820:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773206.888820:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773206.952762:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773206.952802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773206.952824:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773206.952866:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773206.952893:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773206.952910:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773207.080797:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773207.144750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773207.208762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773207.272761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773207.272795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773207.272809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773207.272839:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773207.336784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773207.336807:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773207.336898:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773207.464762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773207.464799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773207.528791:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773207.592796:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773207.656751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773207.656793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773207.656798:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773207.720797:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773207.784848:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773207.784860:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773207.784903:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773207.912789:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773207.912821:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773207.912822:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773207.912822:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773207.976755:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773207.976804:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773207.976826:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773207.976830:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773207.976855:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773207.976882:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773208.104779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773208.168759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773208.232800:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773208.232818:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773208.232820:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773208.232833:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773208.232854:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2204 00000100:00080000:1.0:1790773208.232888:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773208.232918:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773208.232995:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534624384/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773208.233076:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773208.233082:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773208.233085:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773208.233092:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2204 00000100:00080000:1.0:1790773208.233099:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773208.233109:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773208.233126:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773208.233135:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534624384:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773208.233164:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534624512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773208.233193:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773208.233201:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773208.233207:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773208.233216:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2204 00000800:00000100:0.0:1790773208.233225:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773208.233230:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773208.233247:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773208.233260:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773208.233285:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773208/real 1790773208] req@ffff9c4efac4fb80 x1877759534624384/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773208.233294:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534624640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773208.233337:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773208.233339:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534624384/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773208.233357:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773208.233367:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773208.233372:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773208.233384:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2204 00000100:00080000:0.0:1790773208.233385:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773208.233398:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534624384:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773208.233402:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773208.233417:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773208.233428:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773208.233436:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534624512:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773208.233453:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534624768/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773208.233473:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773208.233475:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773208.233486:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773208.233491:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773208.233504:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2204 00000100:00100000:0.0:1790773208.233506:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773208/real 1790773208] req@ffff9c4efac4df80 x1877759534624512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773208.233510:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773208.233525:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773208.233537:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773208.233566:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534624896/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773208.233567:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534624512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773208.233624:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773208.233630:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773208.233633:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773208.233637:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773208.233643:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2204 00000100:00080000:1.0:1790773208.233690:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773208.233700:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773208.233707:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790773208.233708:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773208.233730:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534624512:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773208.233736:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534625024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773208.233757:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773208.233758:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534624640:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773208.233773:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773208.233779:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773208.233789:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2204 00000100:00080000:1.0:1790773208.233794:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773208.233814:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773208.233817:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773208.233828:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773208.233842:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773208/real 1790773208] req@ffff9c4efac4dc00 x1877759534624640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773208.233852:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534625152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773208.233900:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773208.233905:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773208.233908:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773208.233916:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534624640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773208.233920:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2204 00000100:00080000:1.0:1790773208.233935:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773208.233954:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773208.233955:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773208.233970:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790773208.233978:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773208.233979:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534624640:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773208.234038:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534625280/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773208.234046:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534624768:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773208.234076:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773208.234083:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773208.234087:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773208.234094:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2204 00000800:00000100:0.0:1790773208.234104:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773208.234107:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773208.234123:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773208.234132:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773208/real 1790773208] req@ffff9c4efac4ed80 x1877759534624768/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773208.234133:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773208.234163:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534625408/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773208.234167:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534624768/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773208.234225:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773208.234233:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773208.234240:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534624768:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773208.234250:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534624896:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773208.234274:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773208.234287:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773208/real 1790773208] req@ffff9c4efac4c700 x1877759534624896/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773208.234303:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534624896/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773208.234322:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773208.234326:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773208.234330:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534624896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773208.234338:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534625024:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773208.234358:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773208.234371:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773208/real 1790773208] req@ffff9c4efac4f800 x1877759534625024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773208.234386:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534625024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773208.234405:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773208.234409:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773208.234414:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534625024:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773208.234422:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534625152:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773208.234441:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773208.234453:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773208/real 1790773208] req@ffff9c4efac4f480 x1877759534625152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773208.234478:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534625152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773208.234489:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773208.234493:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773208.234497:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534625152:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773208.234516:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534625280:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773208.234536:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773208.234548:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773208/real 1790773208] req@ffff9c4efac4e680 x1877759534625280/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773208.234563:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534625280/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773208.234576:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773208.234580:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773208.234584:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534625280:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773208.234592:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534625408:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773208.234611:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773208.234623:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773208/real 1790773208] req@ffff9c4efac4ea00 x1877759534625408/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773208.234638:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534625408/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773208.234688:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773208.234697:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773208.234705:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534625408:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773208.296753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773208.296783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773208.296789:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773208.296818:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773208.360733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773208.360861:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773208.360988:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773208.488729:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773208.488767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773208.552781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773208.616804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773208.680778:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773208.680782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773208.680842:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773208.744766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773208.808813:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773208.808814:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773208.808829:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773208.936800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773208.936800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773208.936844:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773208.936865:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773209.000772:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773209.000783:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773209.000826:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773209.000839:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773209.000860:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773209.000869:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773209.128751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773209.192756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773209.256723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773209.320720:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773209.320733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773209.320733:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773209.320738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773209.384805:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773209.384860:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773209.384868:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773209.512791:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773209.512799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773209.576755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773209.640816:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773209.704739:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773209.704768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773209.704786:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773209.768772:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773209.832782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773209.832789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773209.832816:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773209.960746:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773209.960763:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773209.960802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773209.960812:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773210.024748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773210.024748:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773210.024748:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773210.024770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773210.024793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773210.024802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773210.152734:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773210.216709:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773210.280707:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773210.344742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773210.344773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773210.344774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773210.344776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773210.408740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773210.408740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773210.408840:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773210.536732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773210.536732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773210.600764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773210.664763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773210.728713:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773210.728726:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773210.728767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773210.792738:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773210.856768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773210.856773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773210.856803:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773210.984759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773210.984763:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773210.984791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773210.984791:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773211.048723:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773211.048728:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773211.048730:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773211.048768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773211.048774:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773211.048784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773211.176698:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773211.240740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773211.304697:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773211.368727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773211.368735:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773211.368735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773211.368737:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773211.432718:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773211.432727:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773211.432784:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773211.560714:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773211.560721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773211.624726:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773211.688716:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773211.752713:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773211.752738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773211.752758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773211.816759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773211.880715:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773211.880718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773211.880725:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773212.008715:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773212.008716:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773212.008719:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773212.008722:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773212.072713:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773212.072719:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773212.072731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773212.072738:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773212.072757:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773212.072766:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773212.200713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773212.264709:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773212.328715:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773212.392735:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773212.392735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773212.392766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773212.392777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773212.456733:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773212.456765:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773212.456780:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773212.584736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773212.584737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773212.648738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773212.712714:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773212.776719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773212.776720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773212.776776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773212.840697:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773212.904732:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773212.904735:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773212.904782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773213.032715:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773213.032728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773213.032743:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773213.032770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773213.096722:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773213.096740:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773213.096745:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773213.096757:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773213.096766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773213.096784:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773213.224718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773213.288781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773213.352777:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773213.352791:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773213.352797:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773213.352801:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773213.352809:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2209 00000100:00080000:1.0:1790773213.352818:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773213.352827:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773213.352860:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534625536/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773213.352881:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773213.352884:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773213.352886:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773213.352889:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2209 00000100:00080000:1.0:1790773213.352891:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773213.352895:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773213.352918:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773213.352930:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534625536:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773213.352931:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534625664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773213.352945:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773213.352948:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773213.352950:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773213.352954:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2209 00000100:00080000:1.0:1790773213.352957:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773213.352960:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773213.352964:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773213.352972:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534625792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773213.352979:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773213.352982:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773213.352984:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773213.352986:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2209 00000100:00080000:1.0:1790773213.352989:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773213.352992:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773213.352996:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773213.352996:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773213.353006:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534625920/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773213.353014:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773213.353016:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773213.353017:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773213.353021:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2209 00000100:00080000:1.0:1790773213.353024:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773213.353027:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773213.353031:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773213.353036:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773213/real 1790773213] req@ffff9c4efac4e300 x1877759534625536/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773213.353043:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534626048/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773213.353052:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773213.353055:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534625536/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773213.353056:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773213.353060:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773213.353068:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2209 00000100:00080000:0.0:1790773213.353071:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773213.353072:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773213.353077:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773213.353081:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790773213.353083:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773213.353090:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534625536:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773213.353095:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534626176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773213.353103:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773213.353106:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773213.353108:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773213.353111:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2209 00000100:00080000:1.0:1790773213.353114:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773213.353120:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773213.353121:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534625664:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790773213.353123:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773213.353134:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534626304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773213.353143:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773213.353146:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773213.353147:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773213.353152:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773213.353156:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2209 00000100:00080000:1.0:1790773213.353160:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773213.353164:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773213.353168:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773213.353169:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773213/real 1790773213] req@ffff9c4efac4ea00 x1877759534625664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773213.353180:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534626432/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773213.353184:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534625664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773213.353202:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773213.353206:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773213.353207:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773213.353211:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2209 00000100:00080000:1.0:1790773213.353213:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773213.353215:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773213.353219:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790773213.353223:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790773213.353227:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534626560/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773213.353230:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773213.353238:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534625664:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773213.353247:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534625792:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773213.353264:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773213.353272:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773213/real 1790773213] req@ffff9c4efac4e680 x1877759534625792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773213.353297:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534625792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773213.353307:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773213.353311:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773213.353315:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534625792:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773213.353321:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534625920:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773213.353339:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773213.353348:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773213/real 1790773213] req@ffff9c4efac4f480 x1877759534625920/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773213.353367:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534625920/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773213.353377:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773213.353380:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773213.353384:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534625920:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773213.353390:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534626048:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773213.353407:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773213.353417:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773213/real 1790773213] req@ffff9c4efac4f800 x1877759534626048/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773213.353429:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534626048/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773213.353438:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773213.353441:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773213.353445:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534626048:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773213.353451:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534626176:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773213.353466:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773213.353475:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773213/real 1790773213] req@ffff9c4efac4c700 x1877759534626176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773213.353488:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534626176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773213.353508:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773213.353512:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773213.353516:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534626176:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773213.353546:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534626304:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773213.353558:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773213.353564:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773213/real 1790773213] req@ffff9c4efac4ed80 x1877759534626304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773213.353572:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534626304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773213.353577:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773213.353579:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773213.353581:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534626304:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773213.353585:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534626432:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773213.353595:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773213.353601:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773213/real 1790773213] req@ffff9c4efac4dc00 x1877759534626432/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773213.353612:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534626432/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773213.353624:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773213.353627:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773213.353632:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534626432:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773213.353639:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534626560:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773213.353660:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773213.353670:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773213/real 1790773213] req@ffff9c4efac4df80 x1877759534626560/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773213.353678:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534626560/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773213.353703:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773213.353706:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773213.353708:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534626560:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773213.416794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773213.416794:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773213.416807:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773213.416861:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773213.480773:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773213.480780:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773213.480796:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773213.608778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773213.608825:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773213.672803:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773213.736787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773213.800731:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773213.800883:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773213.800971:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773213.864711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773213.928707:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773213.928783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773213.928792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773214.056744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773214.056786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773214.056786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773214.056795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773214.120716:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773214.120722:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773214.120724:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773214.120849:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773214.120943:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773214.120966:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773214.248712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773214.312758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773214.376733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773214.440764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773214.440823:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773214.440823:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773214.440831:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773214.504749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773214.504835:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773214.504917:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773214.632794:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773214.632827:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773214.696766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773214.760774:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773214.824785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773214.824812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773214.824812:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773214.888798:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773214.952779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773214.952812:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773214.952812:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773215.080756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773215.080769:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773215.080803:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773215.080843:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773215.144793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773215.144804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773215.144825:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773215.144825:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773215.144858:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773215.144894:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773215.272768:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773215.336752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773215.400770:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773215.464768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773215.464772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773215.464774:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773215.464798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773215.528750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773215.528834:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773215.529034:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773215.656765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773215.656773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773215.720740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773215.784785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773215.848748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773215.848774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773215.848781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773215.912744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773215.976771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773215.976783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773215.976820:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773216.104752:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773216.104780:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773216.104782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773216.104805:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773216.168762:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773216.168800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773216.168803:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773216.168807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773216.168825:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773216.168877:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773216.296736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773216.360727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773216.424761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773216.488782:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773216.488782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773216.488795:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773216.488832:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773216.552814:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773216.552823:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773216.552959:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773216.680730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773216.680739:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773216.744723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773216.808715:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773216.872704:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773216.872708:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773216.872723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773216.936708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773217.000739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773217.000765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773217.000768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773217.128795:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773217.128807:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773217.128825:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773217.128851:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773217.192766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773217.192805:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773217.192811:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773217.192885:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773217.192891:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773217.192895:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773217.320713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773217.384702:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773217.448699:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773217.512760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773217.512764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773217.512782:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773217.512805:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773217.576749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773217.576798:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773217.576874:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773217.704748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773217.704803:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773217.768745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773217.832777:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773217.896745:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773217.896775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773217.896788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773217.960878:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773218.024751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773218.024779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773218.024794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773218.152804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773218.152804:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773218.152846:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773218.152850:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773218.216799:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773218.216838:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773218.216848:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773218.216860:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773218.216906:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773218.216917:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773218.344765:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773218.408762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773218.472764:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773218.472780:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773218.472791:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773218.472798:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773218.472845:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2214 00000100:00080000:1.0:1790773218.472862:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773218.472883:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773218.472959:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534626688/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773218.473007:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773218.473015:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773218.473019:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773218.473028:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2214 00000100:00080000:1.0:1790773218.473036:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773218.473046:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773218.473065:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773218.473092:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534626688:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773218.473101:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534626816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773218.473131:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773218.473137:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773218.473141:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773218.473148:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2214 00000100:00080000:1.0:1790773218.473172:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773218.473190:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773218.473200:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773218.473205:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773218.473233:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534626944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773218.473247:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773218.473253:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773218.473257:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773218.473264:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2214 00000100:00080000:1.0:1790773218.473273:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773218.473275:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773218/real 1790773218] req@ffff9c4efac4fb80 x1877759534626688/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773218.473284:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773218.473297:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773218.473320:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534627072/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773218.473338:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773218.473345:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1790773218.473350:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534626688/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773218.473351:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773218.473368:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2214 00000100:00080000:1.0:1790773218.473377:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773218.473391:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773218.473394:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773218.473398:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:0.0:1790773218.473406:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773218.473418:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534627200/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773218.473420:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534626688:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773218.473461:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773218.473467:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773218.473471:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773218.473481:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534626816:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773218.473482:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2214 00000100:00080000:1.0:1790773218.473495:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773218.473503:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773218.473512:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773218.473533:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534627328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773218.473547:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773218.473565:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773218.473571:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773218.473574:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773218.473582:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773218/real 1790773218] req@ffff9c4efac4df80 x1877759534626816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773218.473586:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2214 00000100:00080000:1.0:1790773218.473597:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773218.473609:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773218.473622:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773218.473634:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534626816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773218.473641:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534627456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773218.473693:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773218.473698:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773218.473701:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773218.473713:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2214 00000100:00080000:1.0:1790773218.473725:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773218.473738:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773218.473747:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773218.473752:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790773218.473764:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534627584/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773218.473786:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773218.473790:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773218.473798:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1790773218.473822:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534626816:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773218.473827:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773218.473845:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2214 00000100:00080000:1.0:1790773218.473850:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773218.473856:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534626944:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773218.473873:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773218.473888:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773218.473908:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534627712/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773218.473938:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773218.473966:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773218/real 1790773218] req@ffff9c4efac4dc00 x1877759534626944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773218.474008:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534626944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773218.474027:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773218.474035:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773218.474044:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534626944:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773218.474058:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534627072:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773218.474095:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773218.474112:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773218/real 1790773218] req@ffff9c4efac4ed80 x1877759534627072/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773218.474128:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534627072/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773218.474142:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773218.474147:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773218.474152:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534627072:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773218.474172:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534627200:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773218.474193:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773218.474205:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773218/real 1790773218] req@ffff9c4efac4c700 x1877759534627200/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773218.474220:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534627200/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773218.474240:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773218.474244:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773218.474251:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534627200:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773218.474285:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534627328:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773218.474325:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773218.474358:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773218/real 1790773218] req@ffff9c4efac4f800 x1877759534627328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773218.474376:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534627328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773218.474397:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773218.474402:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773218.474407:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534627328:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773218.474415:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534627456:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773218.474442:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773218.474456:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773218/real 1790773218] req@ffff9c4efac4f480 x1877759534627456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773218.474482:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534627456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773218.474492:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773218.474496:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773218.474501:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534627456:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773218.474508:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534627584:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773218.474527:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773218.474549:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773218/real 1790773218] req@ffff9c4efac4e680 x1877759534627584/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773218.474564:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534627584/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773218.474577:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773218.474581:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773218.474585:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534627584:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773218.474592:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534627712:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773218.474616:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773218.474636:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773218/real 1790773218] req@ffff9c4efac4ea00 x1877759534627712/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773218.474703:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534627712/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773218.474723:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773218.474728:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773218.474733:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534627712:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773218.536730:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773218.536776:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773218.536860:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773218.536861:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773218.600751:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773218.600757:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773218.600857:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773218.728762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773218.728778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773218.792753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773218.856760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773218.920751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773218.920783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773218.920792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773218.984741:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773219.048748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773219.048773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773219.048830:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773219.176751:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773219.176818:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773219.176821:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773219.176827:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773219.240768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773219.240777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773219.240829:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773219.240843:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773219.240867:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773219.240895:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773219.368756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773219.432759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773219.496863:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773219.560762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773219.560778:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773219.560804:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773219.560845:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773219.624817:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773219.624862:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773219.624878:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773219.752816:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773219.752840:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773219.816729:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773219.880805:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773219.944723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773219.944753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773219.944810:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773220.008806:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773220.072817:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773220.072827:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773220.072831:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773220.200757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773220.200757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773220.200757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773220.200757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773220.264764:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773220.264815:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773220.264844:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773220.264845:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773220.264855:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773220.264879:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773220.392697:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773220.456700:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773220.520746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773220.584738:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773220.584792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773220.584821:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773220.584832:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773220.648835:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773220.648857:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773220.648872:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773220.776798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773220.776814:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773220.840731:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773220.904721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773220.968708:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773220.968711:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773220.968721:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773221.032816:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773221.096813:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773221.096817:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773221.096824:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773221.224704:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773221.224708:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773221.224719:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773221.224719:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773221.288789:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773221.288814:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773221.288849:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773221.288868:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773221.288890:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773221.288931:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773221.416706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773221.480700:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773221.544698:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773221.608719:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773221.608728:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773221.608776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773221.608786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773221.672728:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773221.672772:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773221.672856:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773221.800798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773221.800799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773221.864718:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773221.928710:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773221.992753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773221.992753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773221.992771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773222.056762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773222.120756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773222.120782:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773222.120787:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773222.248769:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773222.248776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773222.248808:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773222.248819:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773222.312813:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773222.312817:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773222.312837:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773222.312849:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773222.312878:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773222.312899:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773222.440723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773222.504723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773222.568746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773222.632767:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773222.632768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773222.632769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773222.632772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773222.696790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773222.696841:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773222.696904:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773222.824746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773222.824813:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773222.888800:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773222.952825:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773223.016784:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773223.016813:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773223.016839:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773223.080743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773223.144749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773223.144756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773223.144805:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773223.272743:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773223.272744:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773223.272828:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773223.272835:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773223.336732:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773223.336750:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773223.336835:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773223.336860:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773223.336861:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773223.336862:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773223.464778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773223.528703:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773223.592749:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773223.592862:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773223.592872:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773223.592880:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773223.592895:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2219 00000100:00080000:3.0:1790773223.592913:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773223.592933:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773223.593015:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534627840/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773223.593044:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773223.593048:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773223.593065:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773223.593070:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2219 00000100:00080000:3.0:1790773223.593074:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773223.593080:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773223.593093:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773223.593104:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534627840:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773223.593113:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534627968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773223.593131:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773223.593134:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773223.593136:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773223.593140:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2219 00000100:00080000:3.0:1790773223.593143:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773223.593147:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773223.593151:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773223.593171:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534628096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773223.593199:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773223.593204:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773223.593206:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773223.593209:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773223.593215:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2219 00000100:00080000:3.0:1790773223.593222:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773223.593230:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773223.593235:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773223.593249:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534628224/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773223.593273:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773223.593280:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773223/real 1790773223] req@ffff9c4ef73c0000 x1877759534627840/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773223.593280:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773223.593293:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773223.593301:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2219 00000100:00100000:0.0:1790773223.593307:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534627840/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773223.593307:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773223.593318:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773223.593324:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:0.0:1790773223.593336:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790773223.593337:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534628352/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773223.593347:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773223.593357:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773223.593359:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534627840:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773223.593365:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773223.593370:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773223.593378:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2219 00000100:00080000:3.0:1790773223.593382:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773223.593387:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773223.593392:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773223.593398:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534627968:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773223.593405:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534628480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773223.593415:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773223.593419:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773223.593422:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773223.593430:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2219 00000800:00000100:0.0:1790773223.593431:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773223.593437:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773223.593444:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773223.593449:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773223.593451:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773223/real 1790773223] req@ffff9c4ef73c1500 x1877759534627968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773223.593463:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534628608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773223.593469:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534627968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773223.593504:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773223.593510:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773223.593512:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773223.593517:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2219 00000100:00080000:3.0:1790773223.593521:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773223.593523:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773223.593527:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773223.593532:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:3.0:1790773223.593535:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773223.593539:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534627968:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773223.593550:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534628736/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773223.593551:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534628096:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773223.593567:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773223.593571:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773223.593574:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773223.593578:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773223.593586:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2219 00000100:00100000:0.0:1790773223.593592:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773223/real 1790773223] req@ffff9c4ef73c0700 x1877759534628096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773223.593593:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773223.593607:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773223.593609:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534628096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773223.593634:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790773223.593637:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773223.593641:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773223.593644:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534628096:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773223.593652:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534628224:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773223.593653:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534628864/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773223.593717:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773223.593732:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773223/real 1790773223] req@ffff9c4ef73c2a00 x1877759534628224/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773223.593770:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534628224/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773223.593787:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773223.593790:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773223.593794:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534628224:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773223.593800:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534628352:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773223.593821:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773223.593830:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773223/real 1790773223] req@ffff9c4ef73c2d80 x1877759534628352/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773223.593840:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534628352/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773223.593848:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773223.593850:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773223.593853:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534628352:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773223.593872:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534628480:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773223.593886:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773223.593894:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773223/real 1790773223] req@ffff9c4ef73c3800 x1877759534628480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773223.593904:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534628480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773223.593918:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773223.593920:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773223.593923:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534628480:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773223.593928:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534628608:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773223.593946:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773223.593983:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773223/real 1790773223] req@ffff9c4ef73c0a80 x1877759534628608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773223.593998:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534628608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773223.594009:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773223.594013:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773223.594017:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534628608:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773223.594023:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534628736:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773223.594065:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773223.594075:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773223/real 1790773223] req@ffff9c4ef73c0e00 x1877759534628736/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773223.594085:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534628736/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773223.594094:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773223.594097:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773223.594100:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534628736:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773223.594104:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534628864:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773223.594117:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773223.594124:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773223/real 1790773223] req@ffff9c4ef73c2300 x1877759534628864/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773223.594133:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534628864/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773223.594141:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773223.594143:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773223.594145:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534628864:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773223.656722:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773223.656724:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773223.656838:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773223.656839:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773223.720869:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773223.720901:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773223.721048:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773223.848811:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773223.848831:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773223.912800:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773223.976826:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773224.040822:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773224.040827:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773224.040842:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773224.104883:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773224.168785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773224.168809:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773224.168813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773224.296811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773224.296832:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773224.296856:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773224.296876:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773224.360817:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773224.360844:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773224.360853:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773224.360865:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773224.360936:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773224.360937:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773224.488770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773224.552785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773224.616794:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773224.680815:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773224.680816:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773224.680833:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773224.680841:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773224.744815:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773224.744849:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773224.744990:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773224.872837:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773224.872855:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773224.936796:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773225.000808:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773225.064769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773225.064771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773225.064813:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773225.128778:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773225.192757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773225.192774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773225.192801:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773225.320809:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773225.320822:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773225.320838:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773225.320848:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773225.384793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773225.384801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773225.384834:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773225.384836:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773225.384865:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773225.384875:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773225.512756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773225.576769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773225.640760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773225.704745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773225.704766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773225.704816:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773225.704823:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773225.768734:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773225.768835:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773225.768972:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773225.896767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773225.896775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773225.960714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773226.024760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773226.088767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773226.088767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773226.088825:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773226.152808:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773226.216796:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773226.216839:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773226.216843:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773226.344849:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773226.344854:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773226.344855:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773226.344856:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773226.408803:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773226.408817:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773226.408845:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773226.408874:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773226.408894:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773226.408900:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773226.536782:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773226.600737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773226.664736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773226.728725:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773226.728749:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773226.728755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773226.728785:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773226.792741:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773226.792791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773226.792884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773226.920711:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773226.920719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773226.984748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773227.048782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773227.112712:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773227.112716:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773227.112719:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773227.176733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773227.240721:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773227.240722:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773227.240724:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773227.368708:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773227.368712:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773227.368715:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773227.368732:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773227.432698:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773227.432714:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773227.432726:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773227.432733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773227.432735:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773227.432749:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773227.560709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773227.624698:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773227.688696:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773227.752714:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773227.752722:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773227.752731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773227.752732:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773227.816730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773227.816781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773227.816783:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773227.944715:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773227.944715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773228.008710:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773228.072746:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773228.136710:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773228.136710:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773228.136738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773228.200711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773228.264707:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773228.264712:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773228.264729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773228.392721:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773228.392721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773228.392724:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773228.392736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773228.456707:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773228.456721:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773228.456723:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773228.456743:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773228.456758:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773228.456762:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773228.584715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773228.648706:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773228.712731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773228.712737:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773228.712743:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773228.712747:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773228.712756:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2224 00000100:00080000:1.0:1790773228.712765:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773228.712776:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773228.712809:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534628992/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773228.712832:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773228.712835:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773228.712837:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773228.712840:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2224 00000100:00080000:1.0:1790773228.712844:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773228.712849:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773228.712858:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773228.712865:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534628992:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773228.712870:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534629120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773228.712880:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773228.712883:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773228.712885:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773228.712888:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2224 00000100:00080000:1.0:1790773228.712891:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773228.712895:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773228.712898:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773228.712907:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534629248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773228.712914:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773228.712917:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773228.712918:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773228.712921:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2224 00000100:00080000:1.0:1790773228.712924:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773228.712925:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773228.712930:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773228.712935:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773228.712944:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534629376/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773228.712956:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773228.712959:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773228.712961:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773228.712966:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773228/real 1790773228] req@ffff9c4efac4e300 x1877759534628992/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773228.712966:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2224 00000100:00080000:1.0:1790773228.712969:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773228.712976:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773228.712984:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773228.712992:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534628992/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773228.712996:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534629504/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773228.713017:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773228.713018:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773228.713024:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773228.713026:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773228.713028:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773228.713033:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534628992:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773228.713034:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2224 00000100:00080000:1.0:1790773228.713040:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773228.713045:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773228.713049:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773228.713056:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534629632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773228.713060:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534629120:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773228.713070:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773228.713074:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773228.713076:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773228.713081:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2224 00000800:00000100:0.0:1790773228.713086:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773228.713088:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773228.713094:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773228.713099:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773228.713102:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773228/real 1790773228] req@ffff9c4efac4ea00 x1877759534629120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773228.713111:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534629760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773228.713116:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534629120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773228.713131:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773228.713136:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773228.713138:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773228.713142:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2224 00000100:00080000:0.0:1790773228.713144:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773228.713145:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773228.713153:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773228.713154:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773228.713161:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773228.713163:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534629120:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773228.713171:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534629248:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773228.713174:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534629888/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773228.713188:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773228.713191:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773228.713196:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1790773228.713199:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773228/real 1790773228] req@ffff9c4efac4e680 x1877759534629248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773228.713200:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773228.713210:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2224 00000100:00080000:1.0:1790773228.713215:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773228.713216:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534629248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773228.713221:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773228.713229:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790773228.713232:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773228.713237:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773228.713242:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534629248:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773228.713242:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534630016/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773228.713253:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534629376:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773228.713275:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773228.713285:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773228/real 1790773228] req@ffff9c4efac4f480 x1877759534629376/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773228.713295:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534629376/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773228.713302:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773228.713304:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773228.713306:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534629376:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773228.713310:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534629504:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773228.713321:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773228.713327:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773228/real 1790773228] req@ffff9c4efac4f800 x1877759534629504/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773228.713335:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534629504/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773228.713342:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773228.713344:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773228.713346:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534629504:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773228.713356:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534629632:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773228.713367:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773228.713373:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773228/real 1790773228] req@ffff9c4efac4c700 x1877759534629632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773228.713381:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534629632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773228.713398:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773228.713401:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773228.713403:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534629632:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773228.713408:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534629760:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773228.713418:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773228.713424:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773228/real 1790773228] req@ffff9c4efac4ed80 x1877759534629760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773228.713432:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534629760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773228.713438:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773228.713440:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773228.713442:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534629760:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773228.713446:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534629888:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773228.713456:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773228.713462:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773228/real 1790773228] req@ffff9c4efac4dc00 x1877759534629888/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773228.713470:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534629888/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773228.713477:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773228.713479:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773228.713482:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534629888:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773228.713486:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534630016:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773228.713496:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773228.713501:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773228/real 1790773228] req@ffff9c4efac4df80 x1877759534630016/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773228.713508:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534630016/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773228.713515:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773228.713517:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773228.713520:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534630016:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773228.776725:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773228.776731:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773228.776731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773228.776772:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773228.840730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773228.840735:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773228.840778:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773228.968698:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773228.968732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773229.032709:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773229.096706:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773229.160700:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773229.160710:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773229.160714:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773229.224722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773229.288714:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773229.288722:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773229.288723:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773229.416708:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773229.416712:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773229.416725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773229.416726:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773229.480725:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773229.480728:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773229.480728:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773229.480731:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773229.480760:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773229.480760:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773229.608707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773229.672711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773229.736716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773229.800706:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773229.800711:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773229.800724:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773229.800727:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773229.864724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773229.864724:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773229.864744:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773229.992713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773229.992724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773230.056721:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773230.120712:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773230.184746:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773230.184763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773230.184785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773230.248714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773230.312798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773230.312799:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773230.312833:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773230.440744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773230.440746:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773230.440751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773230.440751:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773230.504764:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773230.504798:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773230.504814:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773230.504837:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773230.504852:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773230.504866:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773230.632746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773230.696718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773230.760743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773230.824741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773230.824741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773230.824749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773230.824754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773230.888746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773230.888753:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773230.888759:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773231.016699:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773231.016719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773231.080706:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773231.144702:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773231.208731:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773231.208766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773231.208766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773231.272707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773231.336734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773231.336735:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773231.336761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773231.464712:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773231.464719:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773231.464734:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773231.464735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773231.528712:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773231.528728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773231.528734:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773231.528737:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773231.528755:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773231.528768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773231.656713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773231.720720:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773231.784718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773231.848724:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773231.848725:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773231.848727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773231.848729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773231.912725:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773231.912742:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773231.912791:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773232.040704:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773232.040709:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773232.104717:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773232.168706:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773232.232725:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773232.232725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773232.232725:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773232.296699:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773232.360714:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773232.360727:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773232.360728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773232.488725:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773232.488725:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773232.488727:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773232.488727:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773232.552717:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773232.552724:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773232.552736:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773232.552748:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773232.552752:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773232.552767:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773232.680724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773232.744724:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773232.808699:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773232.872711:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773232.872718:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773232.872724:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773232.872730:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773232.936711:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773232.936724:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773232.936784:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773233.064704:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773233.064753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773233.128740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773233.192703:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773233.256716:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773233.256721:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773233.256724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773233.320719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773233.384746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773233.384746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773233.384746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773233.512711:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773233.512725:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773233.512728:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773233.512745:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773233.576713:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773233.576717:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773233.576724:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773233.576735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773233.576759:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773233.576764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773233.704725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773233.768705:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773233.832723:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773233.832731:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1790773233.832731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773233.832736:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773233.832746:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2229 00000100:00080000:0.0:1790773233.832755:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773233.832765:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773233.832798:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534630144/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773233.832821:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773233.832824:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773233.832826:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773233.832829:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2229 00000100:00080000:0.0:1790773233.832832:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773233.832836:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773233.832844:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773233.832851:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534630144:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773233.832856:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534630272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773233.832865:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773233.832869:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773233.832870:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773233.832874:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2229 00000100:00080000:0.0:1790773233.832877:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773233.832879:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773233.832883:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773233.832891:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534630400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773233.832902:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773233.832905:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773233.832907:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773233.832912:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2229 00000800:00000100:1.0:1790773233.832914:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773233.832916:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773233.832921:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773233.832925:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773233.832934:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534630528/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773233.832941:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773233.832944:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773233.832946:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773233.832950:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2229 00000100:00080000:0.0:1790773233.832954:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773233.832954:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773233/real 1790773233] req@ffff9c4ec7a7df80 x1877759534630144/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773233.832962:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773233.832969:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773233.832982:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534630144/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773233.832983:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534630656/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773233.833015:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773233.833017:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773233.833020:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773233.833023:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773233.833024:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773233.833030:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2229 00000100:00100000:1.0:1790773233.833033:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534630144:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773233.833035:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773233.833043:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773233.833046:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773233.833053:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534630784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773233.833060:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534630272:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773233.833063:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773233.833068:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773233.833069:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773233.833073:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2229 00000100:00080000:0.0:1790773233.833080:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773233.833084:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773233.833086:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773233.833092:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773233.833100:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773233/real 1790773233] req@ffff9c4ec7a7f480 x1877759534630272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773233.833103:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534630912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773233.833124:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773233.833126:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534630272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773233.833127:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773233.833131:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773233.833139:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2229 00000100:00080000:0.0:1790773233.833144:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773233.833148:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773233.833152:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790773233.833153:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773233.833161:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773233.833165:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534631040/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773233.833168:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534630272:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773233.833181:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534630400:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773233.833195:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773233.833198:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773233.833200:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773233.833204:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773233.833205:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2229 00000100:00080000:0.0:1790773233.833211:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773233.833216:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773233.833220:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773233/real 1790773233] req@ffff9c4ec7a7c380 x1877759534630400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773233.833222:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773233.833235:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534631168/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773233.833239:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534630400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773233.833256:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773233.833260:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773233.833263:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534630400:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773233.833268:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534630528:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773233.833281:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773233.833288:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773233/real 1790773233] req@ffff9c4ec7a7d880 x1877759534630528/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773233.833295:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534630528/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773233.833303:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773233.833305:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773233.833308:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534630528:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773233.833312:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534630656:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773233.833323:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773233.833329:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773233/real 1790773233] req@ffff9c4ec7a7ed80 x1877759534630656/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773233.833336:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534630656/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773233.833342:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773233.833344:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773233.833346:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534630656:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773233.833365:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534630784:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773233.833378:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773233.833385:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773233/real 1790773233] req@ffff9c4ec7a7ca80 x1877759534630784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773233.833397:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534630784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773233.833409:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773233.833411:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773233.833413:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534630784:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773233.833418:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534630912:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773233.833428:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773233.833442:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773233/real 1790773233] req@ffff9c4ec7a7e300 x1877759534630912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773233.833462:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534630912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773233.833472:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773233.833476:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773233.833480:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534630912:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773233.833488:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534631040:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773233.833524:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773233.833534:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773233/real 1790773233] req@ffff9c4ec7a7fb80 x1877759534631040/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773233.833543:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534631040/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773233.833551:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773233.833554:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773233.833556:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534631040:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773233.833560:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534631168:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773233.833573:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773233.833579:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773233/real 1790773233] req@ffff9c4ec7a7c700 x1877759534631168/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773233.833586:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534631168/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773233.833597:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773233.833600:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773233.833602:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534631168:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773233.896746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773233.896746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773233.896746:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773233.896746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773233.960713:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773233.960734:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773233.960816:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773234.088719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773234.088719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773234.152734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773234.216726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773234.280750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773234.280750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773234.280750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773234.344723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773234.408733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773234.408734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773234.408736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773234.536730:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773234.536730:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773234.536736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773234.536745:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773234.600721:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773234.600724:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773234.600729:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773234.600731:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773234.600755:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773234.600765:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773234.728742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773234.792749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773234.856727:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773234.920744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773234.920809:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773234.920809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773234.920815:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773234.984731:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773234.984735:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773234.984865:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773235.112748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773235.112767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773235.176746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773235.240729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773235.304722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773235.304737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773235.304756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773235.368738:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773235.432729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773235.432733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773235.432756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773235.560725:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773235.560731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773235.560736:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773235.560738:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773235.624710:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773235.624714:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773235.624728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773235.624753:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773235.624758:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773235.624766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773235.752737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773235.816728:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773235.880713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773235.944732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773235.944732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773235.944733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773235.944735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773236.008740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773236.008750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773236.008785:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773236.136720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773236.136720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773236.200728:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773236.264711:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773236.328735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773236.328735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773236.328735:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773236.392734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773236.456711:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773236.456729:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773236.456731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773236.584718:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773236.584720:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773236.584736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773236.584765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773236.648714:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773236.648729:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773236.648739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773236.648739:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773236.648759:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773236.648762:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773236.776731:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773236.840761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773236.904736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773236.968707:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773236.968722:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773236.968722:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773236.968739:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773237.032724:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773237.032725:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773237.032788:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773237.160716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773237.160717:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773237.224726:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773237.288716:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773237.352722:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773237.352724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773237.352725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773237.416710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773237.480774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773237.480774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773237.480774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773237.608739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773237.608740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773237.608740:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773237.608747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773237.672720:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773237.672724:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773237.672736:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773237.672742:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773237.673695:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773237.673725:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773237.800730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773237.864726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773237.928716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773237.992769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773237.992769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773237.992769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773237.992769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773238.056721:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773238.056754:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773238.056799:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773238.184711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773238.184735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773238.248789:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773238.312731:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773238.376753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773238.376754:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773238.376754:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773238.440752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773238.504714:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773238.504723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773238.504739:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773238.632741:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773238.632743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773238.632743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773238.632745:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773238.696720:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773238.696720:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773238.696726:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773238.696768:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773238.696772:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773238.696792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773238.824726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773238.888760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773238.952778:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773238.952793:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773238.952801:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773238.952805:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773238.952816:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2234 00000100:00080000:1.0:1790773238.952831:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773238.952847:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773238.952899:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534631296/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773238.952934:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773238.952938:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773238.952941:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773238.952948:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2234 00000100:00080000:1.0:1790773238.952954:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773238.952963:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773238.952973:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534631296:192.168.204.150@tcp:250:kworker.0 00000008:00100000:1.0:1790773238.952977:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773238.952999:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534631424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773238.953012:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773238.953017:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773238.953020:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773238.953024:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2234 00000100:00080000:1.0:1790773238.953039:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773238.953047:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773238.953049:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773238.953061:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773238.953081:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534631552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773238.953095:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773238.953099:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773238/real 1790773238] req@ffff9c4efac4fb80 x1877759534631296/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773238.953101:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773238.953106:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773238.953123:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2234 00000100:00080000:1.0:1790773238.953131:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773238.953133:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534631296/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773238.953143:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773238.953148:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773238.953153:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00080000:0.0:1790773238.953154:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773238.953165:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534631296:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773238.953183:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534631680/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773238.953193:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534631424:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773238.953209:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773238.953215:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773238.953218:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773238.953225:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773238.953233:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2234 00000100:00100000:0.0:1790773238.953239:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773238/real 1790773238] req@ffff9c4efac4df80 x1877759534631424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773238.953240:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773238.953256:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773238.953262:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534631424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773238.953267:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:0.0:1790773238.953279:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773238.953284:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773238.953291:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534631808/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773238.953293:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534631424:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773238.953316:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534631552:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773238.953337:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773238.953343:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:0.0:1790773238.953347:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773238.953348:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773238.953359:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2234 00000100:00100000:0.0:1790773238.953360:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773238/real 1790773238] req@ffff9c4efac4dc00 x1877759534631552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773238.953371:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773238.953381:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534631552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773238.953386:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773238.953422:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773238.953426:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773238.953431:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773238.953434:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534631552:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773238.953443:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534631936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773238.953450:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534631680:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773238.953482:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773238.953489:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773238.953491:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773238.953496:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773238.953501:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773238/real 1790773238] req@ffff9c4efac4ed80 x1877759534631680/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773238.953506:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2234 00000100:00080000:1.0:1790773238.953519:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773238.953525:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534631680/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773238.953534:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773238.953541:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773238.953545:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773238.953547:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773238.953556:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534631680:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773238.953562:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534631808:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773238.953573:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534632064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773238.953583:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773238.953596:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773238/real 1790773238] req@ffff9c4efac4c700 x1877759534631808/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773238.953601:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773238.953611:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773238.953613:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773238.953626:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534631808/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773238.953627:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2234 00000100:00080000:1.0:1790773238.953639:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773238.953642:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773238.953647:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773238.953679:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534631808:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773238.953679:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773238.953696:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773238.953703:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534631936:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773238.953721:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534632192/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773238.953726:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773238.953739:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773238/real 1790773238] req@ffff9c4efac4f800 x1877759534631936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773238.953747:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773238.953753:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534631936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773238.953754:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773238.953759:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773238.953780:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2234 00000100:00080000:0.0:1790773238.953787:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773238.953788:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773238.953797:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773238.953806:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773238.953807:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534631936:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790773238.953825:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773238.953832:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534632064:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773238.953847:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534632320/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773238.953855:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773238.953867:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773238/real 1790773238] req@ffff9c4efac4f480 x1877759534632064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773238.953879:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534632064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773238.953885:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773238.953889:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773238.953892:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534632064:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773238.953907:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534632192:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773238.953918:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773238.953925:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773238/real 1790773238] req@ffff9c4efac4e680 x1877759534632192/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773238.953933:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534632192/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773238.953940:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773238.953942:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773238.953944:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534632192:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773238.953948:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534632320:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773238.953958:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773238.953964:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773238/real 1790773238] req@ffff9c4efac4ea00 x1877759534632320/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773238.953971:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534632320/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773238.953979:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773238.953982:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773238.953986:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534632320:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773239.016708:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773239.016721:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773239.016722:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773239.016724:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773239.080739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773239.080762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773239.080783:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773239.208705:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773239.208715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773239.272709:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773239.336733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773239.400724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773239.400725:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773239.400725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773239.464708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773239.528731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773239.528731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773239.528731:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773239.656710:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773239.656724:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773239.656726:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773239.656727:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773239.720716:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773239.720716:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773239.720721:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773239.720745:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773239.720760:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773239.720772:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773239.848707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773239.912708:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773239.976719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773240.040725:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773240.040725:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773240.040726:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773240.040727:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773240.104727:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773240.104727:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773240.104761:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773240.232720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773240.232721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773240.296742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773240.360726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773240.424741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773240.424742:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773240.424742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773240.488734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773240.552726:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773240.552726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773240.552726:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773240.680747:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773240.680747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773240.680748:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773240.680748:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773240.744704:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773240.744718:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773240.744728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773240.744731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773240.744745:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773240.744756:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773240.872724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773240.936727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773241.000707:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773241.064738:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773241.064739:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773241.064739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773241.064744:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773241.128752:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773241.128797:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773241.128837:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773241.256708:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773241.256714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773241.320751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773241.384737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773241.448727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773241.448727:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773241.448727:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773241.512696:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773241.576722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773241.576723:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773241.576726:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773241.704727:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773241.704727:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773241.704731:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773241.704737:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773241.768725:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773241.768725:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773241.768728:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773241.768738:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773241.768764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773241.768783:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773241.896712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773241.960706:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773242.024723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773242.088747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773242.088747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773242.088747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773242.088747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773242.152725:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773242.152728:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773242.152779:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773242.280699:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773242.280709:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773242.344710:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773242.408718:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773242.472765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773242.472765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773242.472772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773242.536755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773242.600714:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773242.600717:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773242.600721:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773242.728726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773242.728730:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773242.728733:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773242.728738:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773242.792738:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773242.792748:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773242.792751:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773242.792775:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773242.792788:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773242.792790:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773242.920705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773242.984712:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773243.048744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773243.112774:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773243.112779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773243.112780:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773243.112784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773243.176717:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773243.176766:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773243.176813:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773243.304700:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773243.304718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773243.368729:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773243.432763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773243.496812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773243.496813:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773243.496815:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773243.560771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773243.624812:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773243.624813:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773243.624814:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773243.752753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773243.752777:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773243.752805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773243.752813:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773243.816727:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773243.816732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773243.816747:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773243.816787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773243.816807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773243.816819:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773243.944714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773244.008717:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773244.072749:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773244.072750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773244.072758:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773244.072762:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773244.072771:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2239 00000100:00080000:0.0:1790773244.072781:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773244.072794:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773244.072829:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534632448/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773244.072853:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773244.072858:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773244.072860:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773244.072865:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2239 00000100:00080000:0.0:1790773244.072869:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773244.072876:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773244.072886:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773244.072894:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534632448:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773244.072900:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534632576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773244.072911:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773244.072915:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773244.072918:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773244.072922:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2239 00000100:00080000:0.0:1790773244.072926:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773244.072945:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773244.072951:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773244.072960:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773244.072962:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534632704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773244.072972:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773244.072991:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773244.072993:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773244.072999:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2239 00000100:00080000:0.0:1790773244.073003:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773244.073009:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773244.073014:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773244.073021:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773244/real 1790773244] req@ffff9c4ec7a7d180 x1877759534632448/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773244.073025:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534632832/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773244.073036:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773244.073039:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773244.073041:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534632448/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773244.073042:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773244.073047:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2239 00000100:00080000:0.0:1790773244.073051:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773244.073053:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773244.073056:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773244.073058:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790773244.073061:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773244.073063:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534632448:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773244.073071:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534632960/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773244.073080:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773244.073085:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1790773244.073085:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534632576:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773244.073087:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773244.073092:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2239 00000100:00080000:0.0:1790773244.073095:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773244.073116:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773244.073122:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773244.073124:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773244.073134:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534633088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773244.073138:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773244/real 1790773244] req@ffff9c4ec7a7c700 x1877759534632576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773244.073146:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773244.073151:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1790773244.073152:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534632576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773244.073153:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773244.073159:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2239 00000100:00080000:0.0:1790773244.073163:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773244.073167:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773244.073171:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790773244.073172:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773244.073176:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773244.073180:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534632576:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773244.073182:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534633216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773244.073188:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534632704:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773244.073193:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773244.073197:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773244.073199:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773244.073205:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2239 00000800:00000100:1.0:1790773244.073208:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773244.073209:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773244.073215:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773244.073221:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773244.073222:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773244/real 1790773244] req@ffff9c4ec7a7fb80 x1877759534632704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773244.073232:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534633344/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773244.073236:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534632704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773244.073244:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773244.073247:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773244.073249:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773244.073251:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773244.073251:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773244.073256:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534632704:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773244.073263:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534632832:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773244.073268:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2239 00000100:00080000:0.0:1790773244.073272:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773244.073278:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773244.073284:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773244.073324:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534633472/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773244.073324:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773244.073337:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773244/real 1790773244] req@ffff9c4ec7a7e300 x1877759534632832/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773244.073350:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534632832/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773244.073361:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773244.073365:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773244.073369:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534632832:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773244.073376:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534632960:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773244.073392:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773244.073400:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773244/real 1790773244] req@ffff9c4ec7a7ca80 x1877759534632960/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773244.073408:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534632960/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773244.073415:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773244.073418:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773244.073420:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534632960:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773244.073431:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534633088:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773244.073443:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773244.073449:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773244/real 1790773244] req@ffff9c4ec7a7ed80 x1877759534633088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773244.073464:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534633088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773244.073476:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773244.073478:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773244.073481:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534633088:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773244.073485:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534633216:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773244.073496:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773244.073506:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773244/real 1790773244] req@ffff9c4ec7a7d880 x1877759534633216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773244.073518:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534633216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773244.073527:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773244.073530:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773244.073535:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534633216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773244.073541:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534633344:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773244.073562:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773244.073571:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773244/real 1790773244] req@ffff9c4ec7a7c380 x1877759534633344/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773244.073580:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534633344/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773244.073588:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773244.073590:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773244.073592:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534633344:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773244.073596:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534633472:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773244.073608:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773244.073614:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773244/real 1790773244] req@ffff9c4ec7a7f480 x1877759534633472/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773244.073622:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534633472/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773244.073628:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773244.073630:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773244.073632:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534633472:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773244.136763:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773244.136763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773244.136763:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773244.136769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773244.200737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773244.200795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773244.200801:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773244.328708:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773244.328722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773244.392757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773244.456731:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773244.520714:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773244.520746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773244.520747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773244.584704:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773244.648734:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773244.648734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773244.648734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773244.776743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773244.776744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773244.776745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773244.776745:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773244.840727:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773244.840735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773244.840743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773244.840773:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773244.840788:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773244.840795:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773244.968716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773245.032730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773245.096729:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773245.160714:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773245.160715:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773245.160719:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773245.160732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773245.224711:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773245.224736:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773245.224793:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773245.352716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773245.352717:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773245.416719:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773245.480723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773245.544700:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773245.544718:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773245.544719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773245.608710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773245.672719:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773245.672721:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773245.672723:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773245.800723:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773245.800727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773245.800730:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773245.800731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773245.864732:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773245.864732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773245.864737:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773245.864741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773245.864766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773245.864769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773245.992715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773246.056722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773246.120716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773246.184727:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773246.184728:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773246.184730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773246.184732:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773246.248718:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773246.248729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773246.248793:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773246.376720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773246.376732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773246.440729:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773246.504711:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773246.568712:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773246.568717:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773246.568720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773246.632711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773246.696734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773246.696735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773246.696738:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773246.824723:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773246.824732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773246.824732:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773246.824736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773246.888724:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773246.888730:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773246.888744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773246.888760:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773246.888772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773246.888778:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773247.016703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773247.080727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773247.144721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773247.208705:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773247.208719:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773247.208726:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773247.208726:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773247.272715:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773247.272758:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773247.272849:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773247.400700:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773247.400708:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773247.464702:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773247.528703:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773247.592711:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773247.592715:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773247.592718:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773247.656725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773247.720705:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773247.720714:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773247.720731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773247.848718:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773247.848719:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773247.848720:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773247.848724:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773247.912715:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773247.912717:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773247.912725:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773247.912733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773247.912748:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773247.912759:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773248.040739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773248.104698:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773248.168699:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773248.232710:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773248.232713:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773248.232730:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773248.232731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773248.296715:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773248.296734:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773248.296782:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773248.424700:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773248.424707:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773248.488715:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773248.552725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773248.616725:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773248.616731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773248.616731:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773248.680727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773248.744712:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773248.744726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773248.744727:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773248.872704:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773248.872706:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773248.872712:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773248.872725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773248.936727:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773248.936752:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773248.936763:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773248.936766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773248.936779:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773248.936813:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773249.064714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773249.128702:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773249.192710:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773249.192737:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773249.192743:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773249.192746:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773249.192754:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2245 00000100:00080000:2.0:1790773249.192763:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773249.192773:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773249.192804:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534633600/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773249.192827:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773249.192831:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773249.192833:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773249.192839:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2245 00000100:00080000:2.0:1790773249.192844:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773249.192851:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773249.192855:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534633600:192.168.204.150@tcp:250:kworker.0 00000008:00100000:2.0:1790773249.192863:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773249.192881:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534633728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773249.192890:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773249.192893:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773249.192894:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773249.192898:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2245 00000100:00080000:2.0:1790773249.192901:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773249.192905:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773249.192909:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773249.192914:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773249.192921:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534633856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773249.192939:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773249.192943:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773249.192946:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773249.192949:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773249/real 1790773249] req@ffff9c4efbcc0380 x1877759534633600/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773249.192954:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2245 00000100:00080000:2.0:1790773249.192962:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773249.192972:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534633600/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773249.192972:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773249.192978:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00080000:1.0:1790773249.192985:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773249.192990:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773249.192993:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534633984/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773249.192997:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534633600:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773249.193010:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773249.193013:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773249.193015:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773249.193019:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2245 00000100:00080000:2.0:1790773249.193022:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773249.193023:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534633728:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773249.193028:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773249.193033:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773249.193043:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534634112/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773249.193048:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773249.193058:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773249.193061:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1790773249.193064:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773249/real 1790773249] req@ffff9c4efbcc3100 x1877759534633728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773249.193064:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773249.193072:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2245 00000100:00080000:2.0:1790773249.193078:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773249.193084:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773249.193088:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534633728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773249.193089:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773249.193102:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534634240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773249.193107:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773249.193109:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773249.193114:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773249.193114:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773249.193116:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773249.193121:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534633728:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773249.193122:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2245 00000100:00080000:2.0:1790773249.193127:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773249.193133:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534633856:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773249.193133:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773249.193140:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773249.193150:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534634368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773249.193157:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773249.193167:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773249.193169:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773249/real 1790773249] req@ffff9c4efbcc0700 x1877759534633856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773249.193175:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773249.193180:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773249.193184:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534633856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773249.193189:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2245 00000100:00080000:1.0:1790773249.193196:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773249.193197:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773249.193201:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773249.193218:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534633856:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773249.193223:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534633984:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773249.193226:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773249.193232:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:1.0:1790773249.193243:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773249.193247:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534634496/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773249.193254:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773249/real 1790773249] req@ffff9c4efbcc2680 x1877759534633984/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773249.193283:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773249.193287:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534633984/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773249.193288:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773249.193292:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773249.193300:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2245 00000100:00080000:2.0:1790773249.193305:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773249.193306:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773249.193310:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773249.193313:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773249.193316:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773249.193318:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534633984:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773249.193327:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534634624/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773249.193348:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534634112:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773249.193370:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773249.193383:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773249/real 1790773249] req@ffff9c4efbcc0a80 x1877759534634112/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773249.193397:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534634112/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773249.193407:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773249.193411:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773249.193413:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534634112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773249.193418:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534634240:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773249.193433:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773249.193440:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773249/real 1790773249] req@ffff9c4efbcc2300 x1877759534634240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773249.193448:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534634240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773249.193459:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773249.193462:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773249.193464:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534634240:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773249.193468:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534634368:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773249.193477:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773249.193483:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773249/real 1790773249] req@ffff9c4efbcc1880 x1877759534634368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773249.193491:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534634368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773249.193497:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773249.193499:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773249.193501:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534634368:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773249.193505:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534634496:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773249.193515:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773249.193520:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773249/real 1790773249] req@ffff9c4efbcc1500 x1877759534634496/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773249.193527:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534634496/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773249.193535:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773249.193537:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773249.193539:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534634496:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773249.193544:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534634624:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773249.193553:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773249.193559:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773249/real 1790773249] req@ffff9c4efbcc1f80 x1877759534634624/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773249.193566:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534634624/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773249.193572:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773249.193575:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773249.193577:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534634624:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773249.256700:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773249.256722:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773249.256728:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773249.256734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773249.320708:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773249.320726:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773249.320797:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773249.448705:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773249.448706:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773249.512700:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773249.576714:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773249.640705:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773249.640716:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773249.640723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773249.704704:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773249.768710:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773249.768715:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773249.768721:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773249.896716:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773249.896716:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773249.896729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773249.896730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773249.960727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773249.960736:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773249.960736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773249.960739:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773249.960760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773249.960764:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773250.088703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773250.152709:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773250.216745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773250.280720:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773250.280725:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773250.280725:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773250.280776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773250.344704:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773250.344734:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773250.344885:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773250.472715:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773250.472727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773250.536739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773250.600715:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773250.664742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773250.664745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773250.664761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773250.728730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773250.792715:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773250.792764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773250.792769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773250.920710:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773250.920711:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773250.920720:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773250.920758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773250.984707:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773250.984729:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773250.984743:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773250.984753:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773250.984774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773250.984780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773251.112714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773251.176721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773251.240810:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773251.304737:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773251.304771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773251.304789:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773251.304794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773251.368726:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773251.368770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773251.368875:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773251.496705:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773251.496709:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773251.560738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773251.624706:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773251.688752:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773251.688764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773251.688765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773251.752732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773251.816744:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773251.816744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773251.816749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773251.944740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773251.944775:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773251.944775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773251.944775:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773252.008745:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773252.008764:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773252.008766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773252.008778:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773252.008807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773252.008808:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773252.136698:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773252.200729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773252.264706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773252.328718:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773252.328720:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773252.328725:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773252.328726:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773252.392763:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773252.392764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773252.392854:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773252.520718:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773252.520720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773252.584760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773252.648718:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773252.712708:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773252.712732:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773252.712732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773252.776732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773252.840760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773252.840760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773252.840766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773252.968728:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773252.968728:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773252.968733:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773252.968736:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773253.032726:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773253.032726:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773253.032726:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773253.032729:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773253.032758:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773253.032759:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773253.160707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773253.224738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773253.288710:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773253.352749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773253.352750:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773253.352750:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773253.352750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773253.416714:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773253.416744:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773253.416774:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773253.544708:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773253.544716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773253.608716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773253.672722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773253.736731:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773253.736731:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773253.736735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773253.800718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773253.864732:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773253.864733:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773253.864733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773253.992754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773253.992755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773253.992755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773253.992756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773254.056729:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773254.056731:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773254.056739:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773254.056750:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773254.056767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773254.056768:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773254.184723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773254.248727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773254.312708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773254.312746:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773254.312751:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773254.312755:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773254.312763:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2250 00000100:00080000:0.0:1790773254.312772:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773254.312783:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773254.312817:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534634752/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773254.312848:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773254.312853:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773254.312855:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773254.312860:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2250 00000100:00080000:0.0:1790773254.312864:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773254.312871:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773254.312882:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773254.312895:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534634880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773254.312904:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773254.312907:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773254.312909:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773254.312912:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2250 00000100:00080000:0.0:1790773254.312915:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773254.312918:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534634752:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773254.312922:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773254.312929:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773254.312938:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534635008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773254.312946:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773254.312950:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773254.312952:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773254.312955:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2250 00000100:00080000:0.0:1790773254.312970:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773254.312974:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773254.312978:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773254.312986:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534635136/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773254.312992:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773254.312996:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773254.312998:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773254.313002:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2250 00000100:00080000:0.0:1790773254.313006:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773254.313010:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534634880:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773254.313012:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773254.313021:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773254.313031:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534635264/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773254.313039:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773254.313042:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773254.313043:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773254.313046:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2250 00000100:00080000:0.0:1790773254.313049:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773254.313052:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534635008:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773254.313057:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773254.313062:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773254.313071:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534635392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773254.313077:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534635136:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773254.313085:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773254.313090:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773254.313092:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773254.313096:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2250 00000100:00080000:0.0:1790773254.313100:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773254.313103:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534635264:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773254.313135:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534635392:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773254.313193:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773254.313199:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773254.313208:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534635520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773254.313222:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773254.313225:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773254.313226:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773254.313230:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2250 00000100:00080000:0.0:1790773254.313234:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773254.313239:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773254.313248:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534635520:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773254.313259:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773254.313276:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534635648/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773254.313293:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773254.313298:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773254.313300:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773254.313305:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534635648:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773254.313308:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2250 00000100:00080000:0.0:1790773254.313315:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773254.313326:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773254.313332:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773254.313343:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534635776/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773254.313371:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534635776:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773254.376715:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773254.376715:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773254.376726:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773254.376733:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773254.440718:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773254.440746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773254.440769:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773254.568730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773254.568753:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773254.632746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773254.696751:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773254.760732:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773254.760739:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773254.760739:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773254.824749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773254.888762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773254.888762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773254.888762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773255.016765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773255.016765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773255.016765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773255.016774:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773255.080732:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773255.080732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773255.080735:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773255.080739:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773255.080763:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773255.080765:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773255.208707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773255.272731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773255.336739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773255.400757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773255.400757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773255.400758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773255.400758:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773255.464729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773255.464755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773255.464765:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773255.592741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773255.592741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773255.656725:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773255.720747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773255.784729:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773255.784761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773255.784761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773255.848730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773255.912733:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773255.912741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773255.912749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773256.040748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773256.040749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773256.040749:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773256.040749:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773256.104719:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773256.104727:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773256.104728:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773256.104737:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773256.104755:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773256.104767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773256.232715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773256.296723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773256.360737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773256.424746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773256.424748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773256.424748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773256.424749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773256.488742:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773256.488742:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773256.488804:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773256.616747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773256.616748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773256.680761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773256.744717:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773256.808722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773256.808728:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773256.808808:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773256.872713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773256.936758:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773256.936758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773256.936758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773257.064732:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773257.064732:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773257.064733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773257.064733:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773257.128716:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773257.128718:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773257.128735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773257.128743:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773257.128760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773257.128765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773257.256711:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773257.320711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773257.384721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:2.0:1790773257.384891:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:2.0:1790773257.384929:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1790773257.384947:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773257.385002:0:2383:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:2.0:1790773257.385017:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773257.385038:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773257.385063:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773257.385072:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:1.0:1790773257.385074:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773254/real 1790773257] req@ffff9c4ec7a7c700 x1877759534635776/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1790773257.385081:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773257.385089:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:1.0:1790773257.385097:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534635776/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1790773257.385097:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:1.0:1790773257.385112:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773257.385119:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773257.385123:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534635776:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773257.385132:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773254/real 1790773257] req@ffff9c4ec7a7fb80 x1877759534635648/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773257.385148:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534635648/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773257.385160:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773257.385164:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773257.385168:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534635648:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773257.385176:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773254/real 1790773257] req@ffff9c4ec7a7e300 x1877759534635520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773257.385189:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534635520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773257.385199:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773257.385203:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773257.385207:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534635520:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773257.385215:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773254/real 1790773257] req@ffff9c4ec7a7ca80 x1877759534635392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773257.385228:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534635392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773257.385247:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773257.385252:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773257.385257:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534635392:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773257.385264:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773254/real 1790773257] req@ffff9c4ec7a7c380 x1877759534635008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773257.385278:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534635008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773257.385286:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773257.385290:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773257.385293:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534635008:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773257.385299:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773254/real 1790773257] req@ffff9c4ec7a7d880 x1877759534635136/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773257.385311:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534635136/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773257.385321:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773257.385325:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773257.385328:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534635136:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773257.385334:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773254/real 1790773257] req@ffff9c4ec7a7ed80 x1877759534635264/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773257.385345:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534635264/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773257.385353:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773257.385357:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773257.385361:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534635264:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773257.385367:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773254/real 1790773257] req@ffff9c4ec7a7df80 x1877759534634752/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773257.385378:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534634752/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773257.385385:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773257.385388:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773257.385392:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534634752:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773257.385398:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773254/real 1790773257] req@ffff9c4ec7a7f480 x1877759534634880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773257.385409:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534634880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773257.385423:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773257.385426:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773257.385429:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534634880:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773257.448737:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773257.448737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773257.448737:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773257.448738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773257.512705:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773257.512753:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773257.512762:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773257.640725:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773257.640726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773257.704707:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773257.768765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773257.832764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773257.832766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773257.832775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773257.896743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773257.960718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773257.960748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773257.960761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773258.088742:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773258.088748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773258.088774:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773258.088801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773258.152744:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773258.152745:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773258.152749:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773258.152761:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773258.152789:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773258.152792:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773258.280715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773258.344719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773258.408700:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773258.472740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773258.472762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773258.472766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773258.472816:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773258.536754:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773258.536775:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773258.536901:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773258.664749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773258.664767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773258.728730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773258.792859:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773258.856729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773258.856781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773258.856781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773258.920754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773258.984777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773258.984783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773258.984788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773259.112770:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773259.112774:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773259.112774:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773259.112774:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773259.176761:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773259.176766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773259.176769:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773259.176782:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773259.176811:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773259.176815:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773259.304699:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773259.368725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773259.432731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773259.432746:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773259.432754:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773259.432758:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773259.432767:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2255 00000100:00080000:3.0:1790773259.432777:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773259.432789:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773259.432823:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534635904/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773259.432850:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773259.432854:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773259.432856:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773259.432862:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2255 00000100:00080000:3.0:1790773259.432867:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773259.432875:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773259.432886:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773259.432890:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534635904:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773259.432906:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534636032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773259.432920:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773259.432923:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773259.432926:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773259.432929:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2255 00000100:00080000:3.0:1790773259.432933:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773259.432938:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773259.432945:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773259.432958:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534636160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773259.432969:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773259.432973:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773259.432979:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773259.432980:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773259.433003:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2255 00000100:00080000:3.0:1790773259.433025:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773259.433033:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773259.433040:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773259.433061:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534636288/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773259.433065:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773259/real 1790773259] req@ffff9c4ef73c1c00 x1877759534635904/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773259.433109:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773259.433115:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773259.433117:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773259.433125:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534635904/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773259.433126:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2255 00000100:00080000:3.0:1790773259.433138:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773259.433151:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773259.433153:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773259.433160:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:1.0:1790773259.433165:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773259.433175:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534635904:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773259.433185:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534636416/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773259.433198:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773259.433202:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773259.433204:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773259.433210:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2255 00000100:00100000:1.0:1790773259.433215:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534636032:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773259.433218:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773259.433233:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773259.433241:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773259.433258:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534636544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773259.433272:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773259.433274:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773259.433283:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773259.433286:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773259.433293:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2255 00000100:00080000:3.0:1790773259.433299:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773259.433301:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773259/real 1790773259] req@ffff9c4ef73c2300 x1877759534636032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773259.433311:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773259.433326:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773259.433335:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534636032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773259.433349:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534636672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773259.433391:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773259.433396:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773259.433398:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773259.433404:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773259.433407:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2255 00000100:00080000:1.0:1790773259.433419:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773259.433419:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773259.433432:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534636032:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773259.433444:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534636160:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773259.433453:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773259.433464:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:1.0:1790773259.433480:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773259.433486:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534636800/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773259.433501:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773259.433505:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773259.433509:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773259/real 1790773259] req@ffff9c4ef73c0e00 x1877759534636160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773259.433510:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773259.433523:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2255 00000100:00080000:3.0:1790773259.433536:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773259.433547:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773259.433555:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534636160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773259.433558:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773259.433580:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790773259.433580:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534636928/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773259.433593:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773259.433601:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534636160:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773259.433610:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534636288:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773259.433639:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773259.433683:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773259/real 1790773259] req@ffff9c4ef73c0a80 x1877759534636288/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773259.433697:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534636288/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773259.433707:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773259.433711:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773259.433716:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534636288:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773259.433723:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534636416:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773259.433746:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773259.433758:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773259/real 1790773259] req@ffff9c4ef73c3800 x1877759534636416/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773259.433772:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534636416/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773259.433782:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773259.433786:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773259.433789:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534636416:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773259.433821:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534636544:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773259.433845:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773259.433858:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773259/real 1790773259] req@ffff9c4ef73c2d80 x1877759534636544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773259.433870:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534636544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773259.433888:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773259.433892:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773259.433896:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534636544:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773259.433902:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534636672:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773259.433923:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773259.433936:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773259/real 1790773259] req@ffff9c4ef73c2a00 x1877759534636672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773259.433948:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534636672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773259.433957:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773259.433961:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773259.433965:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534636672:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773259.433972:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534636800:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773259.433996:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773259.434033:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773259/real 1790773259] req@ffff9c4ef73c0700 x1877759534636800/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773259.434045:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534636800/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773259.434057:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773259.434063:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773259.434068:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534636800:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773259.434113:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534636928:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773259.434140:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773259.434152:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773259/real 1790773259] req@ffff9c4ef73c1500 x1877759534636928/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773259.434172:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534636928/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773259.434185:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773259.434190:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773259.434194:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534636928:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773259.496755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773259.496756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773259.496790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773259.496794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773259.560731:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773259.560796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773259.560900:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773259.688750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773259.688786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773259.752785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773259.816777:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773259.880731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773259.880733:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773259.880761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773259.944701:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773260.008723:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773260.008731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773260.008734:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773260.136722:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773260.136742:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773260.136749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773260.136749:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773260.200720:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773260.200733:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773260.200737:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773260.200762:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773260.200764:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773260.200779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773260.328715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773260.392743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773260.456714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773260.520713:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773260.520722:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773260.520732:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773260.520733:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773260.584712:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773260.584741:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773260.584816:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773260.712728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773260.712737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773260.776727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773260.840718:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773260.904708:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773260.904711:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773260.904734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773260.968713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773261.032763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773261.032763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773261.032766:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773261.160749:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773261.160795:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773261.160802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773261.160802:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773261.224725:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773261.224749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773261.224784:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773261.224797:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773261.224816:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773261.224829:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773261.352752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773261.416801:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773261.480746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773261.544748:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773261.544766:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773261.545705:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773261.545708:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773261.608784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773261.608820:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773261.608849:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773261.736737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773261.736737:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773261.800798:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773261.864753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773261.928761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773261.928761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773261.928761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773261.992748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773262.056742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773262.056743:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773262.056743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773262.184750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773262.184750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773262.184751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773262.184751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773262.248748:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773262.248755:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773262.248757:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773262.248766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773262.248795:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773262.248816:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773262.376741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773262.440785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773262.504723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773262.568737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773262.568740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773262.568744:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773262.568747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773262.632753:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773262.632754:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773262.632986:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773262.760732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773262.760735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773262.824776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773262.888753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773262.952771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773262.952776:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773262.952785:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773263.016742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773263.080760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773263.080761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773263.080788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773263.208743:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773263.208744:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773263.208764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773263.208785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773263.272725:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773263.272784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773263.272791:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773263.272813:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773263.272821:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773263.272848:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773263.400755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773263.464743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773263.528723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773263.592764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773263.592770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773263.592771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773263.592786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773263.656748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773263.656749:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773263.656898:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773263.784736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773263.784740:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773263.848772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773263.912766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773263.976710:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773263.976718:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773263.976767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773264.040754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773264.104782:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773264.104805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773264.104805:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773264.232757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773264.232757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773264.232787:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773264.232849:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773264.296758:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773264.296762:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773264.296769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773264.296819:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773264.296842:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773264.296854:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773264.424713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773264.488742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773264.552713:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773264.552724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773264.552724:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773264.552729:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773264.552742:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2260 00000100:00080000:3.0:1790773264.552755:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773264.552769:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773264.552817:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534637056/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773264.552843:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773264.552846:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773264.552848:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773264.552851:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2260 00000100:00080000:3.0:1790773264.552854:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773264.552859:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773264.552872:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773264.552884:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534637056:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773264.552886:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534637184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773264.552898:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773264.552901:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773264.552903:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773264.552925:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2260 00000100:00080000:3.0:1790773264.552928:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773264.552933:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773264.552937:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773264.552947:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534637312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773264.552954:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773264.552957:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773264.552958:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773264.552961:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2260 00000100:00080000:3.0:1790773264.552964:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773264.552967:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773264.552970:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773264.552976:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534637440/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773264.552983:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773264.552985:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773264.552987:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773264.552990:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2260 00000100:00080000:3.0:1790773264.552993:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773264.552996:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773264.552998:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773264.553004:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534637568/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773264.553010:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773264.553012:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773264.553013:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773264.553016:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2260 00000100:00080000:3.0:1790773264.553019:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773264.553022:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773264.553025:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773264.553031:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534637696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773264.553036:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773264.553039:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773264.553040:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773264.553044:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2260 00000100:00080000:3.0:1790773264.553052:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773264.553056:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773264.553060:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773264.553070:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534637824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773264.553077:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773264.553080:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773264.553082:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773264.553085:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2260 00000100:00080000:3.0:1790773264.553088:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773264.553091:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773264.553094:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773264.553102:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534637952/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773264.553108:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773264.553114:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773264.553114:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773264.553119:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773264.553122:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2260 00000100:00080000:3.0:1790773264.553125:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773264.553129:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773264.553132:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773264.553141:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534638080/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773264.553163:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773264/real 1790773264] req@ffff9c4ef73c0000 x1877759534637056/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773264.553179:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534637056/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773264.553189:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773264.553193:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773264.553197:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534637056:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773264.553218:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534637184:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773264.553235:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773264.553244:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773264/real 1790773264] req@ffff9c4ef73c1500 x1877759534637184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773264.553253:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534637184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773264.553267:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773264.553270:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773264.553273:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534637184:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773264.553277:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534637312:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773264.553288:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773264.553294:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773264/real 1790773264] req@ffff9c4ef73c0700 x1877759534637312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773264.553302:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534637312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773264.553308:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773264.553310:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773264.553312:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534637312:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773264.553316:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534637440:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773264.553328:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773264.553334:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773264/real 1790773264] req@ffff9c4ef73c2a00 x1877759534637440/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773264.553342:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534637440/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773264.553354:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773264.553356:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773264.553358:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534637440:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773264.553362:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534637568:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773264.553372:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773264.553377:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773264/real 1790773264] req@ffff9c4ef73c2d80 x1877759534637568/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773264.553385:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534637568/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773264.553392:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773264.553395:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773264.553397:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534637568:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773264.553400:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534637696:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773264.553411:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773264.553417:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773264/real 1790773264] req@ffff9c4ef73c3800 x1877759534637696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773264.553425:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534637696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773264.553437:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773264.553439:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773264.553442:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534637696:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773264.553447:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534637824:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773264.553458:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773264.553464:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773264/real 1790773264] req@ffff9c4ef73c0a80 x1877759534637824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773264.553472:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534637824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773264.553477:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773264.553479:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773264.553482:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534637824:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773264.553486:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534637952:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773264.553496:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773264.553502:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773264/real 1790773264] req@ffff9c4ef73c0e00 x1877759534637952/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773264.553510:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534637952/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773264.553517:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773264.553519:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773264.553521:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534637952:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773264.553524:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534638080:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773264.553535:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773264.553542:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773264/real 1790773264] req@ffff9c4ef73c2300 x1877759534638080/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773264.553549:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534638080/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773264.553556:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773264.553558:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773264.553560:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534638080:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773264.616725:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773264.616725:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773264.616772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773264.616775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773264.680761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773264.680783:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773264.680814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773264.808716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773264.808716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773264.872745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773264.936758:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773265.000732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773265.000732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773265.000746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773265.064757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773265.128739:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773265.128739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773265.128757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773265.256735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773265.256737:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773265.256737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773265.256762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773265.320740:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773265.320751:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773265.320751:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773265.320791:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773265.320800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773265.320801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773265.448723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773265.512739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773265.576706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773265.640748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773265.640748:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773265.640748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773265.640748:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773265.704734:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773265.704762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773265.704765:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773265.832747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773265.832761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773265.896733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773265.960761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773266.024726:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773266.024726:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773266.024732:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773266.088762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773266.152753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773266.152768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773266.152769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773266.280735:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773266.280742:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773266.280766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773266.280766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773266.344739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773266.344742:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773266.344742:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773266.344745:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773266.344812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773266.344814:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773266.472752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773266.536757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773266.600716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773266.664729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773266.664730:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773266.664730:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773266.664785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773266.728769:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773266.728811:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773266.728838:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773266.856734:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773266.856734:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773266.920712:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773266.984761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773267.048743:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773267.048743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773267.048772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773267.112757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773267.176723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773267.176770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773267.176770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773267.304760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773267.304760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773267.304765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773267.304765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773267.368720:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773267.368731:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773267.368736:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773267.368751:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773267.368763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773267.368770:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773267.496721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773267.560722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773267.624747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773267.688719:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773267.688721:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773267.688722:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773267.688722:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773267.752734:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773267.752778:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773267.752802:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773267.880736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773267.880736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773267.944740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773268.008758:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773268.072734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773268.072754:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773268.072765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773268.136731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773268.200753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773268.200767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773268.200771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773268.328756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773268.328759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773268.328760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773268.328767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773268.392749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773268.392750:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773268.392750:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773268.392750:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773268.392784:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773268.392786:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773268.520757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773268.584738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773268.648754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773268.712745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773268.712764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773268.712764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773268.712793:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773268.776757:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773268.776757:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773268.776851:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773268.904728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773268.904739:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773268.968733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773269.032760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773269.096716:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773269.096765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773269.096765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773269.160713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773269.224711:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773269.224751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773269.224753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773269.352743:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773269.352743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773269.352753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773269.352780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773269.416748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773269.416757:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773269.416758:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773269.416758:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773269.416797:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773269.416805:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773269.544720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773269.608719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773269.672731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773269.672750:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773269.672761:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773269.672766:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773269.672777:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2265 00000100:00080000:0.0:1790773269.672788:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773269.672801:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773269.672852:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534638208/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773269.672881:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773269.672886:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773269.672889:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773269.672895:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2265 00000100:00080000:0.0:1790773269.672900:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773269.672906:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773269.672916:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534638208:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773269.672917:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773269.672939:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534638336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773269.672960:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773269.672964:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773269.672967:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773269.672972:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2265 00000100:00080000:0.0:1790773269.672977:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773269.672983:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773269.672986:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773269.672991:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773269.673007:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534638464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773269.673018:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773269.673023:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773269.673025:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773269.673033:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2265 00000100:00100000:1.0:1790773269.673034:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773269/real 1790773269] req@ffff9c4ec7a7d180 x1877759534638208/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773269.673037:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773269.673046:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773269.673051:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773269.673067:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534638592/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773269.673074:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534638208/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773269.673089:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773269.673094:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773269.673095:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773269.673100:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773269.673101:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773269.673109:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534638208:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773269.673113:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2265 00000100:00080000:0.0:1790773269.673119:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773269.673125:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773269.673131:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773269.673133:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534638336:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773269.673151:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534638720/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773269.673160:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773269.673169:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773269.673175:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1790773269.673175:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773269/real 1790773269] req@ffff9c4ec7a7f480 x1877759534638336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773269.673181:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773269.673192:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2265 00000100:00100000:1.0:1790773269.673193:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534638336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773269.673200:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773269.673210:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773269.673211:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790773269.673217:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773269.673217:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773269.673221:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534638336:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773269.673228:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534638464:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773269.673236:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534638848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773269.673244:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773269.673254:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773269.673255:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773269/real 1790773269] req@ffff9c4ec7a7df80 x1877759534638464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773269.673265:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773269.673270:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773269.673272:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534638464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773269.673279:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2265 00000100:00080000:1.0:1790773269.673283:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773269.673286:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773269.673288:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773269.673294:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534638464:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773269.673298:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773269.673301:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534638592:192.168.204.150@tcp:8:kworker.0 00000008:00100000:0.0:1790773269.673311:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773269.673318:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773269.673332:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534638976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773269.673332:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773269/real 1790773269] req@ffff9c4ec7a7ed80 x1877759534638592/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773269.673353:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534638592/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773269.673385:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773269.673387:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773269.673389:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773269.673396:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1790773269.673396:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534638592:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773269.673401:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773269.673408:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2265 00000100:00080000:0.0:1790773269.673413:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773269.673419:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534638720:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773269.673422:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773269.673431:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773269.673446:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534639104/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773269.673449:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773269.673463:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773269.673466:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773269/real 1790773269] req@ffff9c4ec7a7d880 x1877759534638720/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773269.673470:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773269.673474:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773269.673485:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2265 00000100:00080000:0.0:1790773269.673491:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773269.673492:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534638720/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773269.673501:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773269.673508:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790773269.673510:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773269.673514:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773269.673519:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534638720:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773269.673528:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534638848:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773269.673529:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534639232/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773269.673550:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773269.673560:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773269/real 1790773269] req@ffff9c4ec7a7c380 x1877759534638848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773269.673568:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534638848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773269.673581:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773269.673583:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773269.673586:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534638848:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773269.673590:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534638976:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773269.673600:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773269.673607:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773269/real 1790773269] req@ffff9c4ec7a7ca80 x1877759534638976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773269.673614:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534638976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773269.673620:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773269.673622:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773269.673624:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534638976:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773269.673633:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534639104:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773269.673644:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773269.673669:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773269/real 1790773269] req@ffff9c4ec7a7e300 x1877759534639104/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773269.673679:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534639104/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773269.673686:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773269.673689:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773269.673691:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534639104:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773269.673695:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534639232:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773269.673707:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773269.673715:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773269/real 1790773269] req@ffff9c4ec7a7fb80 x1877759534639232/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773269.673723:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534639232/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773269.673730:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773269.673732:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773269.673734:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534639232:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773269.736757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773269.736762:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773269.736785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773269.736788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773269.800749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773269.800816:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773269.800954:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773269.928727:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773269.928737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773269.992749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773270.056758:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773270.120721:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773270.120762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773270.120762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773270.184736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773270.248730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773270.248738:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773270.248739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773270.376731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773270.376777:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773270.376789:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773270.376789:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773270.440779:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773270.440780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773270.440785:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773270.440787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773270.440834:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773270.440848:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773270.568728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773270.632741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773270.696707:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773270.760714:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773270.760744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773270.760744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773270.760760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773270.824738:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773270.824745:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773270.824864:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773270.952741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773270.952750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773271.016757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773271.080723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773271.144755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773271.144756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773271.144764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773271.208717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773271.272754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773271.272754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773271.272755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773271.400750:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773271.400750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773271.400750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773271.400751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773271.464715:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773271.464729:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773271.464736:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773271.464741:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773271.464757:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773271.464787:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773271.592714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773271.656718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773271.720737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773271.784744:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773271.784747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773271.784753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773271.784754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773271.848733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773271.848774:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773271.848781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773271.976704:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773271.976728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773272.040738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773272.104720:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773272.168707:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773272.168727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773272.168728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773272.232696:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773272.296718:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773272.296718:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773272.296727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773272.424721:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773272.424722:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773272.424722:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773272.424722:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773272.488704:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773272.488717:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773272.488723:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773272.488731:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773272.488742:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773272.488771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773272.616713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773272.680708:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773272.744733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773272.808706:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773272.808723:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773272.808727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773272.808728:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773272.872741:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773272.872759:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773272.872816:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773273.000711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773273.000714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773273.064733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773273.128736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773273.192700:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773273.192704:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773273.192721:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773273.256703:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773273.320725:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773273.320725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773273.320727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773273.448719:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773273.448721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773273.448729:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773273.448729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773273.512723:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773273.512728:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773273.512753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773273.512753:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773273.512777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773273.512783:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773273.640709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773273.704719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773273.768706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773273.832708:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773273.832720:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773273.832742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773273.832746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773273.896709:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773273.896770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773273.896795:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773274.024709:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773274.024713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773274.088709:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773274.152733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773274.216709:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773274.216710:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773274.216732:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773274.280747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773274.344759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773274.344759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773274.344765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773274.472738:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773274.472739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773274.472748:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773274.472750:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773274.536737:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773274.536740:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773274.536742:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773274.536764:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773274.536774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773274.536789:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773274.664797:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773274.728812:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773274.792793:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773274.792832:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773274.792838:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773274.792843:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773274.792851:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2270 00000100:00080000:3.0:1790773274.792867:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773274.792880:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773274.792920:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534639360/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773274.792946:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773274.792949:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773274.792951:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773274.792954:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2270 00000100:00080000:3.0:1790773274.792958:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773274.792963:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773274.792972:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773274.792985:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534639488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773274.792994:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773274.792996:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534639360:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773274.792997:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773274.793001:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773274.793008:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2270 00000100:00080000:3.0:1790773274.793011:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773274.793014:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773274.793017:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773274.793027:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534639616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773274.793034:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773274.793037:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773274.793038:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773274.793042:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2270 00000100:00080000:3.0:1790773274.793045:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773274.793048:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773274.793052:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773274.793060:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534639744/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773274.793063:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773274.793072:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773274.793076:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773274.793077:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773274.793082:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2270 00000100:00080000:3.0:1790773274.793084:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773274.793087:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773274.793090:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773274.793100:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534639872/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773274.793107:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773274/real 1790773274] req@ffff9c4ef73c1c00 x1877759534639360/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773274.793109:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773274.793116:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773274.793119:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773274.793127:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2270 00000100:00080000:3.0:1790773274.793130:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773274.793136:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534639360/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773274.793136:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773274.793143:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773274.793154:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790773274.793157:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534640000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773274.793164:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773274.793169:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773274.793171:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773274.793172:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773274.793178:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534639360:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773274.793179:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2270 00000100:00080000:3.0:1790773274.793187:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773274.793190:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773274.793194:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773274.793201:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534640128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773274.793207:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534639488:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773274.793214:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773274.793218:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773274.793220:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773274.793224:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2270 00000100:00080000:3.0:1790773274.793227:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773274.793233:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773274.793234:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790773274.793241:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773274.793249:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773274/real 1790773274] req@ffff9c4ef73c2300 x1877759534639488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773274.793253:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534640256/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773274.793275:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773274.793276:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534639488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773274.793279:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773274.793282:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773274.793291:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2270 00000100:00080000:3.0:1790773274.793295:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773274.793298:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773274.793301:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773274.793309:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534640384/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773274.793319:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773274.793326:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773274.793332:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534639488:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773274.793340:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534639616:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773274.793362:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773274.793372:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773274/real 1790773274] req@ffff9c4ef73c0e00 x1877759534639616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773274.793381:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534639616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773274.793387:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773274.793390:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773274.793392:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534639616:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773274.793397:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534639744:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773274.793409:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773274.793415:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773274/real 1790773274] req@ffff9c4ef73c0a80 x1877759534639744/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773274.793423:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534639744/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773274.793435:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773274.793437:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773274.793440:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534639744:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773274.793444:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534639872:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773274.793454:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773274.793460:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773274/real 1790773274] req@ffff9c4ef73c3800 x1877759534639872/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773274.793467:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534639872/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773274.793473:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773274.793475:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773274.793478:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534639872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773274.793481:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534640000:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773274.793491:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773274.793498:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773274/real 1790773274] req@ffff9c4ef73c2d80 x1877759534640000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773274.793505:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534640000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773274.793517:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773274.793519:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773274.793522:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534640000:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773274.793536:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534640128:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773274.793547:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773274.793553:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773274/real 1790773274] req@ffff9c4ef73c2a00 x1877759534640128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773274.793561:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534640128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773274.793567:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773274.793569:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773274.793571:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534640128:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773274.793588:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534640256:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773274.793608:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773274.793620:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773274/real 1790773274] req@ffff9c4ef73c0700 x1877759534640256/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773274.793633:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534640256/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773274.793644:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773274.793647:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773274.793673:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534640256:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773274.793679:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534640384:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773274.793695:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773274.793704:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773274/real 1790773274] req@ffff9c4ef73c1500 x1877759534640384/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773274.793728:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534640384/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773274.793735:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773274.793737:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773274.793740:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534640384:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773274.856783:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773274.856830:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773274.856880:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773274.856891:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773274.920783:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773274.920824:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773274.920903:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773275.048796:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773275.048817:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773275.112768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773275.176735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773275.240717:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773275.240734:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773275.240765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773275.304724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773275.368765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773275.368765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773275.368798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773275.496774:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773275.496775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773275.496775:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773275.496784:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773275.560773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773275.560776:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773275.560782:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773275.560786:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773275.560825:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773275.560828:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773275.688722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773275.752753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773275.816747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773275.880757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773275.880757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773275.880761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773275.880770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773275.944748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773275.944764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773275.944830:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773276.072775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773276.072790:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773276.136793:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773276.200832:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773276.264734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773276.264742:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773276.264744:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773276.328744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773276.392801:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773276.392809:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773276.392815:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773276.520770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773276.520772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773276.520775:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773276.520775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773276.584727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773276.584729:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773276.584768:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773276.584769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773276.584781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773276.584783:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773276.712774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773276.776782:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773276.840766:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773276.904821:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773276.904822:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773276.904830:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773276.904848:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773276.968760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773276.968824:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773276.968945:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773277.096768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773277.096804:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773277.160805:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773277.224838:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773277.288755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773277.288796:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773277.288799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773277.352821:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773277.416799:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773277.416799:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773277.416837:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773277.544745:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773277.544808:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773277.544810:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773277.544811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773277.608723:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773277.608764:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773277.608770:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773277.608806:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773277.608810:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773277.608845:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773277.736744:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773277.800758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773277.864777:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773277.928796:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773277.928802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773277.928811:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773277.928842:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773277.992810:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773277.992908:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773277.993071:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773278.120727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773278.120843:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773278.184727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773278.248772:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773278.312714:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773278.312714:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773278.312727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773278.376788:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773278.440742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773278.440763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773278.440763:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773278.568815:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773278.568841:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773278.568852:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773278.568854:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773278.632744:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773278.632756:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773278.632761:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773278.632785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773278.632793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773278.632807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773278.760723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773278.824753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773278.888721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773278.952755:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773278.952755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773278.952760:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773278.952764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773279.016754:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773279.016766:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773279.016808:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773279.144743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773279.144761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773279.208807:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773279.272823:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773279.336812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773279.336812:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773279.336813:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773279.400763:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773279.464808:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773279.464856:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773279.464875:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773279.592796:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773279.592796:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773279.592805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773279.592991:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773279.656738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773279.656744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773279.656771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773279.656778:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773279.656814:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773279.656818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773279.784728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773279.848748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773279.912729:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773279.912737:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773279.912742:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773279.912750:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2275 00000100:00100000:1.0:1790773279.912752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773279.912761:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773279.912772:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773279.912806:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534640512/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773279.912830:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773279.912835:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773279.912837:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773279.912840:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2275 00000100:00080000:3.0:1790773279.912842:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773279.912846:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773279.912854:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773279.912864:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534640640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773279.912870:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534640512:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773279.912876:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773279.912880:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773279.912881:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773279.912885:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2275 00000100:00080000:3.0:1790773279.912887:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773279.912890:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773279.912894:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773279.912902:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534640768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773279.912909:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773279.912912:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773279.912913:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773279.912915:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2275 00000100:00080000:3.0:1790773279.912918:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773279.912922:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773279.912925:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773279.912933:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534640896/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773279.912939:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773279.912942:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773279.912945:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773279.912947:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773279.912950:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2275 00000100:00080000:3.0:1790773279.912954:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773279.912957:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773279.912961:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773279.912969:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534641024/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773279.912975:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773279.912977:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773279.912979:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773279.912982:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2275 00000100:00080000:3.0:1790773279.912984:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773279.912988:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773279.912991:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773279.912999:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773279/real 1790773279] req@ffff9c4ef73c0000 x1877759534640512/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773279.913000:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534641152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773279.913022:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773279.913029:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773279.913031:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773279.913041:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534640512/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773279.913042:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2275 00000100:00080000:3.0:1790773279.913048:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773279.913054:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773279.913063:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773279.913065:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773279.913073:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790773279.913074:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534641280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773279.913086:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534640512:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773279.913093:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773279.913098:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773279.913100:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773279.913106:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2275 00000100:00080000:3.0:1790773279.913111:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773279.913116:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534640640:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773279.913120:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773279.913128:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773279.913143:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534641408/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773279.913152:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773279.913158:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773279.913161:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773279.913163:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773279.913167:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2275 00000100:00080000:3.0:1790773279.913171:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773279.913175:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773279/real 1790773279] req@ffff9c4ef73c1500 x1877759534640640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773279.913176:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773279.913182:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773279.913196:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534641536/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773279.913201:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534640640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773279.913224:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773279.913228:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773279.913232:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534640640:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773279.913239:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534640768:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773279.913260:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773279.913272:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773279/real 1790773279] req@ffff9c4ef73c0700 x1877759534640768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773279.913283:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534640768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773279.913293:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773279.913296:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773279.913300:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534640768:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773279.913307:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534640896:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773279.913328:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773279.913339:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773279/real 1790773279] req@ffff9c4ef73c2a00 x1877759534640896/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773279.913351:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534640896/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773279.913362:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773279.913366:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773279.913369:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534640896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773279.913375:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534641024:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773279.913406:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773279.913416:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773279/real 1790773279] req@ffff9c4ef73c2d80 x1877759534641024/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773279.913429:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534641024/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773279.913438:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773279.913442:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773279.913446:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534641024:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773279.913452:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534641152:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773279.913471:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773279.913482:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773279/real 1790773279] req@ffff9c4ef73c3800 x1877759534641152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773279.913494:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534641152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773279.913511:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773279.913515:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773279.913518:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534641152:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773279.913525:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534641280:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773279.913544:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773279.913555:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773279/real 1790773279] req@ffff9c4ef73c0a80 x1877759534641280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773279.913566:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534641280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773279.913575:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773279.913579:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773279.913582:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534641280:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773279.913604:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534641408:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773279.913643:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773279.913687:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773279/real 1790773279] req@ffff9c4ef73c0e00 x1877759534641408/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773279.913701:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534641408/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773279.913712:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773279.913715:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773279.913720:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534641408:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773279.913726:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534641536:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773279.913746:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773279.913757:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773279/real 1790773279] req@ffff9c4ef73c2300 x1877759534641536/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773279.913769:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534641536/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773279.913780:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773279.913783:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773279.913787:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534641536:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773279.976736:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773279.976736:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773279.976736:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773279.976736:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773280.040737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773280.040742:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773280.040864:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773280.168733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773280.168744:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773280.232753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773280.296749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773280.360752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773280.360752:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773280.360752:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773280.424766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773280.488746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773280.488762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773280.488768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773280.616737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773280.616753:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773280.616766:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773280.616772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773280.680756:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773280.680816:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773280.680821:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773280.680838:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773280.680840:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773280.680886:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773280.808771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773280.872723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773280.936719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773281.000732:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773281.000738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773281.000757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773281.000766:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773281.064735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773281.064742:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773281.064853:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773281.192726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773281.192768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773281.256775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773281.320750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773281.384725:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773281.384754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773281.384759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773281.448724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773281.512713:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773281.512721:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773281.512764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773281.640795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773281.640795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773281.640796:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773281.640815:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773281.704730:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773281.704758:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773281.704762:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773281.704764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773281.704783:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773281.704813:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773281.832723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773281.896796:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773281.960793:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773282.024747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773282.024755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773282.024784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773282.024785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773282.088809:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773282.088870:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773282.089086:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773282.216752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773282.216808:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773282.280753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773282.344809:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773282.408751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773282.408758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773282.408760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773282.472747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773282.536778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773282.536818:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773282.536836:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773282.664750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773282.664759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773282.664802:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773282.664810:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773282.728844:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773282.728850:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773282.728850:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773282.728864:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773282.728943:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773282.728947:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773282.856759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773282.920742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773282.984767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773283.048798:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773283.048817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773283.048851:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773283.048855:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773283.112798:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773283.112831:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773283.112905:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773283.240779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773283.240780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773283.304774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773283.368747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773283.432771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773283.432815:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773283.432816:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773283.496747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773283.560770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773283.560771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773283.560785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773283.688758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773283.688796:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773283.688801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773283.688843:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773283.752821:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773283.752835:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773283.752837:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773283.752837:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773283.752904:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773283.752910:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773283.880746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773283.944767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773284.008825:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773284.072753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773284.072762:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773284.072767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773284.072831:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773284.136776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773284.136833:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773284.136896:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773284.264707:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773284.264712:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773284.328711:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773284.392732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773284.456766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773284.456766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773284.456773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773284.520734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773284.584780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773284.584798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773284.584801:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773284.712772:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773284.712778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773284.712808:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773284.712843:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773284.776831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773284.776834:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773284.776845:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773284.776886:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773284.776925:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773284.776954:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773284.904787:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773284.968814:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773285.032745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773285.032790:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773285.032804:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773285.032811:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773285.032825:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2280 00000100:00080000:3.0:1790773285.032840:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773285.032860:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773285.032945:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534641664/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773285.032989:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773285.032993:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773285.032996:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773285.033011:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2280 00000100:00080000:3.0:1790773285.033016:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773285.033041:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773285.033054:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773285.033094:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534641792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773285.033108:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534641664:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773285.033113:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773285.033125:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773285.033138:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773285.033144:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2280 00000100:00080000:3.0:1790773285.033148:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773285.033153:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773285.033158:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773285.033172:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534641920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773285.033182:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773285.033187:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773285.033190:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773285.033195:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2280 00000100:00080000:3.0:1790773285.033201:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773285.033220:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773285.033220:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773285.033254:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773285.033271:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534642048/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773285.033282:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773285.033286:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773285.033290:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773285.033295:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773285/real 1790773285] req@ffff9c4ef73c1c00 x1877759534641664/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773285.033302:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2280 00000100:00080000:3.0:1790773285.033313:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773285.033327:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773285.033342:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773285.033377:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534642176/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773285.033400:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534641664/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773285.033403:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773285.033420:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773285.033424:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773285.033439:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2280 00000100:00080000:3.0:1790773285.033447:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773285.033454:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773285.033457:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773285.033461:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773285.033476:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790773285.033480:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534642304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773285.033496:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534641664:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773285.033567:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773285.033571:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773285.033574:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773285.033581:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2280 00000100:00080000:3.0:1790773285.033585:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773285.033592:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773285.033596:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534641792:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790773285.033601:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773285.033631:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534642432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773285.033644:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773285.033648:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773285.033651:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773285.033659:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2280 00000800:00000100:1.0:1790773285.033667:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773285.033714:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773285.033728:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773285.033739:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773285.033742:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773285/real 1790773285] req@ffff9c4ef73c2300 x1877759534641792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773285.033784:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534642560/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773285.033786:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534641792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773285.033815:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773285.033822:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773285.033825:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773285.033833:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773285.033841:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773285.033842:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534641792:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773285.033868:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534641920:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773285.033868:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2280 00000100:00080000:3.0:1790773285.033887:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773285.033902:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773285.033909:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790773285.033914:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773285.033924:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773285/real 1790773285] req@ffff9c4ef73c0e00 x1877759534641920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773285.033935:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534642688/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773285.033942:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534641920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773285.033986:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773285.033990:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773285.033994:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534641920:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773285.034005:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534642048:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773285.034035:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773285.034047:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773285/real 1790773285] req@ffff9c4ef73c0a80 x1877759534642048/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773285.034057:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534642048/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773285.034066:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773285.034069:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773285.034071:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534642048:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773285.034077:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534642176:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773285.034091:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773285.034099:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773285/real 1790773285] req@ffff9c4ef73c3800 x1877759534642176/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773285.034109:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534642176/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773285.034116:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773285.034119:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773285.034123:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534642176:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773285.034128:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534642304:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773285.034142:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773285.034150:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773285/real 1790773285] req@ffff9c4ef73c2d80 x1877759534642304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773285.034159:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534642304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773285.034172:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773285.034175:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773285.034179:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534642304:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773285.034199:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534642432:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773285.034214:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773285.034222:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773285/real 1790773285] req@ffff9c4ef73c2a00 x1877759534642432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773285.034232:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534642432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773285.034239:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773285.034242:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773285.034245:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534642432:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773285.034250:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534642560:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773285.034263:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773285.034284:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773285/real 1790773285] req@ffff9c4ef73c0700 x1877759534642560/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773285.034293:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534642560/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773285.034302:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773285.034305:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773285.034308:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534642560:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773285.034313:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534642688:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773285.034326:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773285.034333:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773285/real 1790773285] req@ffff9c4ef73c1500 x1877759534642688/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773285.034343:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534642688/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773285.034368:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773285.034371:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773285.034375:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534642688:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773285.096794:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773285.096794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773285.096800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773285.096813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773285.160863:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773285.160864:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773285.160904:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773285.288753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773285.288756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773285.352796:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773285.416789:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773285.480752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773285.480791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773285.480843:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773285.544785:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773285.608815:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773285.608816:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773285.608860:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773285.736805:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773285.736811:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773285.736908:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773285.736909:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773285.800776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773285.800776:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773285.800777:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773285.800827:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773285.800867:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773285.800873:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773285.928743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773285.992736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773286.056726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773286.120767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773286.120767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773286.120767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773286.120805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773286.184745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773286.184771:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773286.184882:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773286.312750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773286.312793:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773286.376795:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773286.440805:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773286.504800:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773286.504800:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773286.504815:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773286.568744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773286.632774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773286.632792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773286.632833:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773286.760781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773286.760791:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773286.760802:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773286.760833:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773286.824746:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773286.824767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773286.824782:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773286.824824:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773286.824856:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773286.824856:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773286.952743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773287.016748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773287.080731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773287.144779:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773287.144812:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773287.144849:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773287.144865:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773287.208754:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773287.208828:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773287.208849:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773287.336809:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773287.336882:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773287.400849:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773287.464846:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773287.528758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773287.528775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773287.528799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773287.592760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773287.656818:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773287.656819:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773287.656827:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773287.784760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773287.784772:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773287.784773:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773287.784790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773287.848806:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773287.848854:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773287.848863:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773287.848875:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773287.848911:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773287.848918:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773287.976770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773288.040784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773288.104780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773288.168770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773288.168783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773288.168822:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773288.168834:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00010000:00010000:0.0:1790773288.232871:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773288.232914:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773288.232925:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773288.360773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773288.360835:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773288.424792:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773288.488782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773288.552792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773288.552804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773288.552812:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773288.616737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773288.680791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773288.680816:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773288.680823:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773288.808751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773288.808752:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773288.808752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773288.808761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773288.872739:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773288.872750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773288.872762:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773288.872764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773288.872799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773288.872801:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773289.000715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773289.064769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773289.128723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773289.192761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773289.192761:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773289.192764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773289.192766:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773289.256746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773289.256758:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773289.256885:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773289.384710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773289.384769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773289.448754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773289.512782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773289.576780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773289.576781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773289.576808:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773289.640808:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773289.704839:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773289.704840:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773289.704850:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773289.832801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773289.832812:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773289.832812:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773289.832846:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773289.896754:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773289.896758:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773289.896776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773289.896794:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773289.896813:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773289.896838:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773290.024714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773290.088723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773290.152730:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773290.152731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773290.152741:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773290.152746:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773290.152756:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2286 00000100:00080000:3.0:1790773290.152768:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773290.152780:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773290.152835:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534642816/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773290.152869:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773290.152873:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773290.152875:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773290.152880:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2286 00000100:00080000:3.0:1790773290.152893:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773290.152901:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773290.152914:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534642816:192.168.204.150@tcp:250:kworker.0 00000008:00100000:3.0:1790773290.152914:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773290.152936:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534642944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773290.152947:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773290.152952:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773290.152954:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773290.152959:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2286 00000100:00080000:3.0:1790773290.152964:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773290.152970:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773290.152976:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773290.152991:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773290.152992:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534643072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773290.153004:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773290.153009:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773290.153012:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773290.153017:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2286 00000100:00080000:3.0:1790773290.153021:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773290.153028:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773290.153034:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773290.153039:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773290/real 1790773290] req@ffff9c4ef73c0000 x1877759534642816/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773290.153051:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534643200/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773290.153056:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534642816/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773290.153083:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773290.153084:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773290.153092:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773290.153093:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773290.153097:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773290.153098:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534642816:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773290.153110:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2286 00000100:00080000:3.0:1790773290.153115:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773290.153120:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773290.153125:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773290.153127:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534642944:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773290.153146:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534643328/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773290.153151:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773290.153164:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773290/real 1790773290] req@ffff9c4ef73c1500 x1877759534642944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773290.153165:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773290.153177:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773290.153181:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773290.153184:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534642944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773290.153190:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2286 00000100:00080000:3.0:1790773290.153199:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773290.153206:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773290.153209:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773290.153211:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773290.153215:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773290.153219:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534642944:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773290.153228:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534643072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773290.153228:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534643456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773290.153242:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773290.153246:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:1.0:1790773290.153250:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773290.153250:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773290.153258:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2286 00000100:00080000:3.0:1790773290.153261:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773290.153263:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773290/real 1790773290] req@ffff9c4ef73c0700 x1877759534643072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773290.153267:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773290.153275:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773290.153287:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534643584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773290.153296:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534643072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773290.153297:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773290.153306:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773290.153309:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773290.153317:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2286 00000100:00080000:1.0:1790773290.153321:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773290.153321:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773290.153325:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773290.153330:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:3.0:1790773290.153331:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773290.153339:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534643072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773290.153344:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534643712/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773290.153349:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534643200:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790773290.153359:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773290.153362:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773290.153364:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773290.153368:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2286 00000100:00080000:3.0:1790773290.153370:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773290.153374:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773290.153378:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:1.0:1790773290.153387:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773290.153390:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534643840/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773290.153420:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773290/real 1790773290] req@ffff9c4ef73c2a00 x1877759534643200/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773290.153432:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534643200/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773290.153440:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773290.153443:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773290.153445:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534643200:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773290.153461:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534643328:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773290.153477:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773290.153484:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773290/real 1790773290] req@ffff9c4ef73c2d80 x1877759534643328/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773290.153493:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534643328/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773290.153504:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773290.153507:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773290.153509:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534643328:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773290.153513:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534643456:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773290.153523:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773290.153530:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773290/real 1790773290] req@ffff9c4ef73c3800 x1877759534643456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773290.153538:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534643456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773290.153550:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773290.153551:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773290.153554:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534643456:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773290.153558:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534643584:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773290.153574:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773290.153598:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773290/real 1790773290] req@ffff9c4ef73c0a80 x1877759534643584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773290.153612:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534643584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773290.153622:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773290.153626:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773290.153630:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534643584:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773290.153637:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534643712:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773290.153683:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773290.153692:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773290/real 1790773290] req@ffff9c4ef73c0e00 x1877759534643712/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773290.153701:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534643712/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773290.153709:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773290.153712:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773290.153714:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534643712:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773290.153719:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534643840:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773290.153731:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773290.153737:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773290/real 1790773290] req@ffff9c4ef73c2300 x1877759534643840/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773290.153745:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534643840/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773290.153752:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773290.153754:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773290.153756:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534643840:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773290.216720:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773290.216744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773290.216744:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773290.216745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773290.280747:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773290.280811:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773290.280833:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773290.408761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773290.408775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773290.472735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773290.536780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773290.600769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773290.600769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773290.600787:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773290.664736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773290.728737:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773290.728764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773290.728797:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773290.856757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773290.856758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773290.856758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773290.856785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773290.920708:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773290.920722:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773290.920728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773290.920798:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773290.920799:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773290.920799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773291.048735:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773291.112753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773291.176715:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773291.240728:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773291.240749:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773291.240759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773291.240760:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773291.304734:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773291.304735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773291.304780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773291.432733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773291.432733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773291.496769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773291.560719:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773291.624750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773291.624777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773291.624778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773291.688731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773291.752728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773291.752747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773291.752835:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773291.880869:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773291.880895:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773291.880899:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773291.880905:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773291.944735:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773291.944735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773291.944800:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773291.944816:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773291.944862:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773291.944900:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773292.072710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773292.136719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773292.200749:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773292.264732:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773292.264732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773292.264745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773292.264746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773292.328744:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773292.328839:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773292.328916:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773292.456744:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773292.456797:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773292.520730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773292.584753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773292.648727:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773292.648746:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773292.648812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773292.712787:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773292.776733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773292.776750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773292.776752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773292.904778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773292.904786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773292.904835:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773292.904837:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773292.968755:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773292.968797:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773292.968803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773292.968830:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773292.968850:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773292.968877:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773293.096749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773293.160743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773293.224738:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773293.288747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773293.288764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773293.288775:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773293.288790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773293.352801:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773293.352859:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773293.352897:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773293.480745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773293.480761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773293.544762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773293.608785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773293.672740:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773293.672744:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773293.672746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773293.736781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773293.800796:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773293.800798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773293.800802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773293.928763:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773293.928763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773293.928818:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773293.928822:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773293.992758:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773293.992790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773293.992797:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773293.992805:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773293.992833:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773293.992870:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773294.120709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773294.184707:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773294.248724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773294.312742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773294.312742:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773294.312742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773294.312748:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773294.376723:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773294.376755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773294.376874:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773294.504741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773294.504753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773294.568715:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773294.632756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773294.696702:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773294.696705:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773294.696739:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773294.760740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773294.824737:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773294.824744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773294.824750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773294.952723:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773294.952724:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773294.952731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773294.952732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773295.016718:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773295.016733:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773295.016737:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773295.016755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773295.016766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773295.016770:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773295.144717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773295.208732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773295.272721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773295.272721:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773295.272730:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773295.272734:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773295.272742:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2291 00000100:00080000:3.0:1790773295.272751:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773295.272761:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773295.272793:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534643968/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773295.272816:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773295.272819:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773295.272820:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773295.272823:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2291 00000100:00080000:3.0:1790773295.272826:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773295.272830:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773295.272838:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773295.272847:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534643968:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773295.272851:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534644096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773295.272861:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773295.272863:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773295.272865:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773295.272868:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2291 00000100:00080000:3.0:1790773295.272871:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773295.272874:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773295.272877:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773295.272885:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534644224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773295.272892:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773295.272896:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773295.272897:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773295.272900:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2291 00000100:00080000:3.0:1790773295.272903:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773295.272907:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773295.272909:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773295.272914:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773295.272923:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534644352/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773295.272930:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773295.272932:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773295.272934:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773295.272937:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2291 00000100:00080000:3.0:1790773295.272941:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773295.272944:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773295/real 1790773295] req@ffff9c4ef73c1c00 x1877759534643968/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773295.272948:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773295.272954:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773295.272966:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534644480/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773295.272968:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534643968/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773295.272990:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773295.272993:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773295.272994:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773295.272998:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773295.273001:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773295.273005:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2291 00000100:00100000:1.0:1790773295.273009:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534643968:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773295.273010:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773295.273017:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773295.273020:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773295.273027:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534644608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773295.273033:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534644096:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773295.273038:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773295.273042:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773295.273043:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773295.273048:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2291 00000100:00080000:3.0:1790773295.273052:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773295.273057:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773295.273057:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773295.273065:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773295.273072:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773295/real 1790773295] req@ffff9c4ef73c2300 x1877759534644096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773295.273076:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534644736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773295.273097:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534644096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773295.273124:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773295.273128:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773295.273130:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773295.273132:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773295.273135:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773295.273142:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2291 00000100:00100000:1.0:1790773295.273143:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534644096:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773295.273150:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773295.273153:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534644224:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773295.273158:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773295.273166:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:1.0:1790773295.273177:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773295.273181:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534644864/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773295.273189:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773295/real 1790773295] req@ffff9c4ef73c0e00 x1877759534644224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773295.273190:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773295.273198:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773295.273202:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773295.273210:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2291 00000100:00100000:1.0:1790773295.273214:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534644224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773295.273216:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773295.273223:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773295.273231:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773295.273232:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773295.273238:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773295.273243:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534644224:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773295.273243:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534644992/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773295.273264:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534644352:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773295.273281:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773295.273288:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773295/real 1790773295] req@ffff9c4ef73c0a80 x1877759534644352/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773295.273295:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534644352/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773295.273303:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773295.273305:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773295.273322:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534644352:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773295.273326:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534644480:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773295.273339:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773295.273346:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773295/real 1790773295] req@ffff9c4ef73c3800 x1877759534644480/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773295.273359:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534644480/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773295.273366:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773295.273369:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773295.273371:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534644480:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773295.273383:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534644608:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773295.273393:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773295.273400:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773295/real 1790773295] req@ffff9c4ef73c2d80 x1877759534644608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773295.273408:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534644608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773295.273419:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773295.273421:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773295.273423:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534644608:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773295.273427:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534644736:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773295.273437:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773295.273443:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773295/real 1790773295] req@ffff9c4ef73c2a00 x1877759534644736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773295.273450:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534644736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773295.273461:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773295.273463:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773295.273465:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534644736:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773295.273469:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534644864:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773295.273478:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773295.273484:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773295/real 1790773295] req@ffff9c4ef73c0700 x1877759534644864/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773295.273490:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534644864/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773295.273497:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773295.273499:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773295.273502:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534644864:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773295.273505:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534644992:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773295.273515:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773295.273520:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773295/real 1790773295] req@ffff9c4ef73c1500 x1877759534644992/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773295.273527:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534644992/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773295.273534:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773295.273536:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773295.273539:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534644992:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773295.336781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773295.336791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773295.336814:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773295.336815:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773295.400715:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773295.400748:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773295.400848:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773295.528742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773295.528743:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773295.592766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773295.656736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773295.720721:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773295.720735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773295.720736:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773295.784722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773295.848736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773295.848737:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773295.848737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773295.976722:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773295.976722:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773295.976723:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773295.976724:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773296.040722:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773296.040732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773296.040732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773296.040738:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773296.040764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773296.040766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773296.168699:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773296.232776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773296.296730:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773296.360735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773296.360742:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773296.360756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773296.360758:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773296.424730:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773296.424731:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773296.424867:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773296.552777:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773296.552835:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773296.616714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773296.680728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773296.744725:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773296.744735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773296.744773:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773296.808703:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773296.872740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773296.872746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773296.872764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773297.000728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773297.000735:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773297.000740:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773297.000748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773297.064763:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773297.064775:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773297.064813:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773297.064817:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773297.064833:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773297.064856:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773297.192715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773297.256742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773297.320736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773297.384718:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773297.384720:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773297.384720:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773297.384730:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773297.448724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773297.448732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773297.448821:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773297.576738:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773297.576738:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773297.640728:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773297.704722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773297.768716:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773297.768717:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773297.768750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773297.832754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773297.896713:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773297.896715:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773297.896726:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773298.024715:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773298.024715:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773298.024724:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773298.024724:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773298.088723:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773298.088723:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773298.088736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773298.088736:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773298.088759:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773298.088776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773298.216704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773298.280722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773298.344703:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773298.408705:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773298.408727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773298.408727:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773298.408728:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773298.472698:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773298.472717:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773298.472792:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773298.600706:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773298.600712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773298.664716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773298.728725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773298.792728:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773298.792728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773298.792731:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773298.856739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773298.920708:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773298.920720:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773298.920721:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773299.048781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773299.048782:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773299.048786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773299.048869:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773299.112713:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773299.112720:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773299.112743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773299.112778:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773299.112780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773299.112800:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773299.240719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773299.304703:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773299.368750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773299.432766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773299.432774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773299.432796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773299.432799:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773299.496739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773299.496780:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773299.496894:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773299.624736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773299.624737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773299.688734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773299.752745:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773299.816725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773299.816741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773299.816741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773299.880705:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773299.944734:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773299.944734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773299.944746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773300.072750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773300.072750:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773300.072777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773300.072783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773300.136727:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773300.136761:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773300.136781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773300.136784:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773300.136785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773300.136825:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773300.264713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773300.328727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773300.392724:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773300.392734:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773300.392738:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773300.392749:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2296 00000100:00100000:1.0:1790773300.392756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773300.392761:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773300.392776:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773300.392810:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534645120/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773300.392834:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773300.392837:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773300.392839:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773300.392843:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2296 00000100:00080000:3.0:1790773300.392856:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773300.392859:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773300.392867:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773300.392880:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534645248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773300.392889:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773300.392894:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534645120:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773300.392895:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773300.392900:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773300.392921:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2296 00000100:00080000:3.0:1790773300.392928:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773300.392933:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773300.392937:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773300.392952:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534645376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773300.392960:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773300.392963:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773300.392964:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773300.392967:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2296 00000100:00080000:3.0:1790773300.392969:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773300.392973:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773300.392977:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773300.392985:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534645504/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773300.392993:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773300.392995:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773300.392996:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773300.392999:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2296 00000100:00080000:3.0:1790773300.393003:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773300.393008:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773300.393012:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773300.393026:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534645632/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773300.393027:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773300.393042:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773300.393046:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773300.393047:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773300.393051:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2296 00000100:00080000:3.0:1790773300.393058:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773300.393066:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773300.393070:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773300.393080:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534645760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773300.393086:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773300.393090:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773300.393092:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773300.393097:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773300/real 1790773300] req@ffff9c4ef73c0000 x1877759534645120/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773300.393103:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2296 00000100:00080000:3.0:1790773300.393117:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773300.393122:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773300.393134:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773300.393149:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534645888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773300.393158:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534645120/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773300.393166:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773300.393184:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773300.393189:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773300.393199:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2296 00000100:00080000:1.0:1790773300.393201:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773300.393202:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773300.393227:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773300.393230:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773300.393236:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534645120:192.168.204.150@tcp:250:kworker.0 00000008:00100000:3.0:1790773300.393238:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773300.393254:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534646016/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773300.393269:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534645248:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773300.393270:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773300.393291:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773300.393295:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773300.393300:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2296 00000100:00080000:3.0:1790773300.393306:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773300.393312:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773300.393316:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:1.0:1790773300.393322:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773300.393332:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534646144/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773300.393342:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773300/real 1790773300] req@ffff9c4ef73c1500 x1877759534645248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773300.393357:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534645248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773300.393375:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773300.393379:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773300.393384:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534645248:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773300.393391:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534645376:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773300.393415:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773300.393429:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773300/real 1790773300] req@ffff9c4ef73c0700 x1877759534645376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773300.393442:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534645376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773300.393449:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773300.393452:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773300.393455:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534645376:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773300.393460:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534645504:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773300.393477:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773300.393485:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773300/real 1790773300] req@ffff9c4ef73c2a00 x1877759534645504/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773300.393493:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534645504/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773300.393500:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773300.393502:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773300.393504:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534645504:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773300.393509:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534645632:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773300.393520:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773300.393527:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773300/real 1790773300] req@ffff9c4ef73c2d80 x1877759534645632/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773300.393535:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534645632/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773300.393541:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773300.393543:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773300.393546:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534645632:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773300.393567:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534645760:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773300.393587:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773300.393597:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773300/real 1790773300] req@ffff9c4ef73c3800 x1877759534645760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773300.393609:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534645760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773300.393626:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773300.393629:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773300.393633:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534645760:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773300.393639:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534645888:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773300.393656:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773300.393697:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773300/real 1790773300] req@ffff9c4ef73c0a80 x1877759534645888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773300.393711:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534645888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773300.393721:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773300.393726:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773300.393729:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534645888:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773300.393757:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534646016:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773300.393778:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773300.393789:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773300/real 1790773300] req@ffff9c4ef73c0e00 x1877759534646016/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773300.393801:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534646016/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773300.393812:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773300.393816:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773300.393819:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534646016:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773300.393826:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534646144:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773300.393856:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773300.393862:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773300/real 1790773300] req@ffff9c4ef73c2300 x1877759534646144/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773300.393870:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534646144/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773300.393877:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773300.393879:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773300.393882:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534646144:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773300.456719:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773300.456723:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773300.456725:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773300.456754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773300.520700:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773300.520725:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773300.520860:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773300.648708:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773300.648710:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773300.712810:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773300.776752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773300.840757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773300.840761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773300.840764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773300.904711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773300.968723:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773300.968729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773300.968729:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773301.096726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773301.096745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773301.096754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773301.096768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773301.160725:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773301.160745:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773301.160777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773301.160781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773301.160791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773301.160797:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773301.288730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773301.352756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773301.416825:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773301.480757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773301.480768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773301.480771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773301.480820:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773301.544858:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773301.544869:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773301.545025:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773301.672772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773301.672820:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773301.736816:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773301.800737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773301.864753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773301.864753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773301.864753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773301.928710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773301.992754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773301.992762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773301.992768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773302.120711:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773302.120717:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773302.120728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773302.120734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773302.184694:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773302.184717:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773302.184723:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773302.184729:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773302.184740:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773302.184754:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773302.312730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773302.376713:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773302.440730:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773302.504714:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773302.504725:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773302.504732:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773302.504751:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773302.568722:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773302.568730:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773302.568878:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773302.696767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773302.696779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773302.760788:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773302.824765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773302.888822:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773302.888823:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773302.888823:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773302.952771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773303.016720:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773303.016782:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773303.016799:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773303.144729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773303.144729:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773303.144803:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773303.144813:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773303.208794:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773303.208821:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773303.208868:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773303.208868:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773303.208929:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773303.208931:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773303.336775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773303.400767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773303.464743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773303.528757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773303.528763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773303.528792:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773303.528806:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773303.592768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773303.592820:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773303.593008:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773303.720749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773303.720750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773303.784747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773303.848794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773303.912805:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773303.912805:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773303.912805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773303.976783:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773304.040750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773304.040750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773304.040768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773304.168809:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773304.168816:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773304.168821:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773304.168825:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773304.232884:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773304.232885:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773304.232885:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773304.232885:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773304.232949:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773304.232970:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773304.360761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773304.424758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773304.488758:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773304.552748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773304.552773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773304.552780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773304.552785:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773304.616770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773304.616869:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773304.616981:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773304.744764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773304.744825:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773304.808796:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773304.872728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773304.936742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773304.936744:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773304.936792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773305.000717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773305.064757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773305.064787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773305.064813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773305.192779:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773305.192779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773305.192820:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773305.192846:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773305.256751:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773305.256751:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773305.256781:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773305.256786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773305.256816:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773305.256834:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773305.384728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773305.448776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773305.512740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773305.512817:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773305.512828:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773305.512834:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773305.512846:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2301 00000100:00080000:2.0:1790773305.512857:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773305.512871:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773305.512914:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534646272/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773305.512960:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773305.512964:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773305.512966:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773305.512972:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2301 00000100:00080000:2.0:1790773305.512976:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773305.512985:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773305.512995:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773305.513007:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534646272:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773305.513014:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534646400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773305.513054:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773305.513059:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773305.513062:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773305.513068:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2301 00000100:00080000:2.0:1790773305.513073:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773305.513084:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773305.513094:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773305.513130:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773305.513172:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534646528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773305.513191:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773305.513196:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773305.513200:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773305.513209:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2301 00000100:00080000:2.0:1790773305.513218:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773305.513239:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773305/real 1790773305] req@ffff9c4efbcc1c00 x1877759534646272/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773305.513246:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773305.513276:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773305.513303:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534646656/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773305.513304:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534646272/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773305.513325:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773305.513337:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773305.513340:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773305.513341:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773305.513370:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773305.513370:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2301 00000100:00080000:2.0:1790773305.513390:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773305.513392:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534646272:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773305.513403:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773305.513409:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773305.513425:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534646784/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773305.513429:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534646400:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773305.513473:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773305.513479:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:1.0:1790773305.513485:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773305.513487:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773305.513498:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2301 00000100:00080000:2.0:1790773305.513504:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773305.513512:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773305/real 1790773305] req@ffff9c4efbcc2a00 x1877759534646400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773305.513512:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773305.513549:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773305.513556:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534646400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773305.513572:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534646912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773305.513578:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773305.513584:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773305.513589:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773305.513594:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773305.513598:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773305.513599:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534646400:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773305.513611:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2301 00000100:00100000:1.0:1790773305.513618:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534646528:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773305.513619:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773305.513633:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773305.513639:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773305.513650:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534647040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773305.513666:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773305.513670:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773305.513673:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773305.513678:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2301 00000800:00000100:1.0:1790773305.513686:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773305.513689:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773305.513706:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773305.513708:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773305/real 1790773305] req@ffff9c4efbcc1f80 x1877759534646528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773305.513718:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534646528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773305.513738:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773305.513742:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773305.513747:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534646528:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773305.513757:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534646656:192.168.204.150@tcp:8:kworker.0 00000008:00100000:2.0:1790773305.513777:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:1.0:1790773305.513784:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773305.513798:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534647168/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773305.513799:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773305/real 1790773305] req@ffff9c4efbcc1500 x1877759534646656/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773305.513838:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773305.513842:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773305.513845:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773305.513848:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534646656/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773305.513853:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2301 00000100:00080000:2.0:1790773305.513862:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773305.513872:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773305.513876:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790773305.513879:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773305.513891:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773305.513900:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534647296/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773305.513901:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534646656:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773305.513956:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534646784:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773305.513978:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773305.513988:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773305/real 1790773305] req@ffff9c4efbcc1880 x1877759534646784/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773305.513996:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534646784/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773305.514004:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773305.514007:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773305.514009:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534646784:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773305.514026:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534646912:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773305.514038:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773305.514044:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773305/real 1790773305] req@ffff9c4efbcc2300 x1877759534646912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773305.514052:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534646912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773305.514064:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773305.514067:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773305.514069:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534646912:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773305.514073:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534647040:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773305.514084:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773305.514090:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773305/real 1790773305] req@ffff9c4efbcc0a80 x1877759534647040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773305.514099:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534647040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773305.514105:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773305.514125:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773305.514128:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534647040:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773305.514143:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534647168:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773305.514160:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773305.514168:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773305/real 1790773305] req@ffff9c4efbcc2680 x1877759534647168/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773305.514176:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534647168/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773305.514184:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773305.514187:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773305.514189:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534647168:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773305.514194:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534647296:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773305.514215:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773305.514221:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773305/real 1790773305] req@ffff9c4efbcc0700 x1877759534647296/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773305.514230:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534647296/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773305.514236:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773305.514238:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773305.514241:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534647296:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773305.576799:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773305.576801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773305.576802:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773305.576806:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773305.640830:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773305.640831:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773305.640876:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773305.768782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773305.768795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773305.832819:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773305.896756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773305.960768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773305.960810:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773305.960811:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773306.024768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773306.088767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773306.088777:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773306.088777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773306.216798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773306.216801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773306.216803:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773306.216815:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773306.280746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773306.280762:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773306.280770:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773306.280781:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773306.280786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773306.280791:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773306.408745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773306.472720:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773306.536717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773306.600736:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773306.600736:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773306.600772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773306.600773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773306.664755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773306.664782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773306.664964:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773306.792743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773306.792747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773306.856802:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773306.920738:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773306.984759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773306.984773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773306.984783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773307.048713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773307.112737:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773307.112746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773307.112759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773307.240716:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773307.240739:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773307.240762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773307.240768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773307.304764:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773307.304791:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773307.304791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773307.304792:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773307.304827:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773307.304837:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773307.432734:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773307.496792:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773307.560729:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773307.624721:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773307.624726:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773307.624733:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773307.624756:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773307.688802:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773307.688826:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773307.688940:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773307.816781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773307.816781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773307.880794:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773307.944750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773308.008785:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773308.008786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773308.008787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773308.072739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773308.136783:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773308.136813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773308.136814:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773308.264768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773308.264768:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773308.264779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773308.264803:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773308.328773:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773308.328773:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773308.328780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773308.328789:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773308.328827:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773308.328837:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773308.456708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773308.520732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773308.584725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773308.648758:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773308.648764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773308.648766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773308.648777:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773308.712768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773308.712804:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773308.712951:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773308.840713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773308.840734:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773308.904812:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773308.968741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773309.032742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773309.032750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773309.032835:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773309.096734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773309.160747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773309.160749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773309.160780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773309.288743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773309.288743:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773309.288743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773309.288807:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773309.352735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773309.352735:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773309.352735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773309.352780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773309.352787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773309.352798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773309.480721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773309.544708:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773309.608713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773309.672832:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773309.672832:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773309.672858:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773309.672894:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773309.736837:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773309.736897:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773309.737173:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773309.864755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773309.864762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773309.928805:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773309.992801:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773310.056776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773310.056850:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773310.056863:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773310.120786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773310.184806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773310.184806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773310.184811:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773310.312777:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773310.312784:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773310.312830:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773310.312840:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773310.376746:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773310.376751:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773310.376792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773310.376816:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773310.376853:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773310.376870:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773310.504705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773310.568715:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773310.632738:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773310.632748:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773310.632771:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773310.632775:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773310.632785:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2306 00000100:00080000:2.0:1790773310.632796:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773310.632807:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773310.632841:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534647424/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773310.632879:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773310.632883:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773310.632884:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773310.632888:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2306 00000100:00080000:2.0:1790773310.632917:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773310.632922:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773310.632931:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773310.632938:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534647424:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773310.632944:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534647552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773310.632954:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773310.632957:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773310.632958:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773310.632961:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2306 00000100:00080000:2.0:1790773310.632964:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773310.632968:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773310.632972:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773310.632995:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534647680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773310.633020:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773310.633025:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773310.633028:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773310.633108:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2306 00000800:00000100:1.0:1790773310.633110:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773310.633117:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773310.633126:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773310.633132:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773310.633147:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534647808/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773310.633159:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773310.633160:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773310/real 1790773310] req@ffff9c4efbcc3100 x1877759534647424/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773310.633165:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773310.633170:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773310.633179:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2306 00000100:00080000:2.0:1790773310.633185:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773310.633209:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773310.633216:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534647424/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773310.633234:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790773310.633234:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:1.0:1790773310.633243:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773310.633249:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534647424:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773310.633274:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534647936/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773310.633293:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773310.633298:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534647552:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773310.633301:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773310.633307:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773310.633343:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2306 00000100:00080000:2.0:1790773310.633350:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773310.633359:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773310.633367:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773310.633397:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534648064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773310.633403:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773310.633422:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773310.633428:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1790773310.633433:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773310/real 1790773310] req@ffff9c4efbcc0380 x1877759534647552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773310.633434:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773310.633459:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2306 00000100:00080000:2.0:1790773310.633467:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773310.633471:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534647552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773310.633481:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773310.633493:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773310.633497:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773310.633504:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773310.633509:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534647552:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773310.633517:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534648192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773310.633523:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534647680:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773310.633546:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773310.633551:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773310.633555:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773310.633557:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773310.633568:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2306 00000100:00100000:1.0:1790773310.633575:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773310/real 1790773310] req@ffff9c4efbcc0700 x1877759534647680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773310.633577:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773310.633592:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773310.633596:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534647680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773310.633603:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773310.633608:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773310.633613:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773310.633633:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534647680:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773310.633640:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534648320/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773310.633641:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534647808:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773310.633672:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773310.633677:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773310.633679:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773310.633701:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2306 00000800:00000100:1.0:1790773310.633707:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773310.633721:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773310/real 1790773310] req@ffff9c4efbcc2680 x1877759534647808/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773310.633742:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534647808/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773310.633753:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773310.633754:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773310.633761:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773310.633768:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534647808:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773310.633770:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773310.633783:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773310.633793:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534647936:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773310.633805:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534648448/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773310.633817:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773310.633831:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773310/real 1790773310] req@ffff9c4efbcc0a80 x1877759534647936/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773310.633854:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534647936/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773310.633863:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773310.633867:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773310.633870:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534647936:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773310.633892:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534648064:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773310.633921:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773310.633930:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773310/real 1790773310] req@ffff9c4efbcc2300 x1877759534648064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773310.633943:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534648064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773310.633958:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773310.633961:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773310.633965:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534648064:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773310.633971:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534648192:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773310.633995:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773310.634005:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773310/real 1790773310] req@ffff9c4efbcc1880 x1877759534648192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773310.634062:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534648192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773310.634072:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773310.634076:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773310.634080:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534648192:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773310.634088:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534648320:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773310.634108:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773310.634117:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773310/real 1790773310] req@ffff9c4efbcc1500 x1877759534648320/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773310.634126:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534648320/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773310.634134:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773310.634137:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773310.634139:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534648320:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773310.634156:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534648448:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773310.634178:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773310.634191:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773310/real 1790773310] req@ffff9c4efbcc1f80 x1877759534648448/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773310.634205:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534648448/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773310.634213:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773310.634216:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773310.634218:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534648448:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773310.696721:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773310.696722:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773310.696733:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773310.696734:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773310.760798:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773310.760809:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773310.760853:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773310.888723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773310.888811:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773310.952787:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773311.016777:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773311.080730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773311.080741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773311.080752:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773311.144708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773311.208752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773311.208787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773311.208791:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773311.336733:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773311.336733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773311.336733:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773311.336769:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773311.400764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773311.400782:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773311.400790:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773311.400791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773311.400848:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773311.400859:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773311.528712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773311.592725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773311.656717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773311.720712:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773311.720739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773311.720740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773311.720740:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773311.784716:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773311.784737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773311.784765:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773311.912766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773311.912769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773311.976764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773312.040735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773312.104759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773312.104760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773312.104760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773312.168758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773312.232754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773312.232754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773312.232754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773312.360775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773312.360775:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773312.360775:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773312.360810:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773312.424785:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773312.424786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773312.424786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773312.424786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773312.424824:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773312.424827:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773312.552779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773312.616762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773312.680720:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773312.744721:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773312.744738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773312.744773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773312.744794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773312.808772:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773312.808772:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773312.808844:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773312.936724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773312.936729:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773313.000822:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773313.064732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773313.128722:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773313.128723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773313.128790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773313.192732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773313.256764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773313.256783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773313.256784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773313.384754:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773313.384793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773313.384798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773313.384809:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773313.448717:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773313.448757:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773313.448760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773313.448764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773313.448789:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773313.448802:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773313.576737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773313.640706:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773313.704729:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773313.768722:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773313.768722:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773313.768728:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773313.768742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773313.832722:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773313.832722:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773313.832775:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773313.960742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773313.960742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773314.024740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773314.088729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773314.152764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773314.152764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773314.152764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773314.216753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773314.280713:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773314.280717:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773314.280737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773314.408791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773314.408791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773314.408793:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773314.408800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773314.472730:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773314.472763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773314.472784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773314.472810:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773314.472812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773314.472836:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773314.600714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773314.664785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773314.728718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773314.792808:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773314.792808:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773314.792809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773314.792811:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773314.856751:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773314.856803:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773314.856843:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773314.984733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773314.984734:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773315.048761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773315.112762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773315.176760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773315.176760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773315.176771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773315.240765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773315.304734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773315.304772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773315.304772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773315.432808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773315.432808:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773315.432809:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773315.432811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773315.496749:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773315.496749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773315.496750:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773315.496758:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773315.496786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773315.496803:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773315.624719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773315.688717:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773315.752771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773315.752776:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773315.752784:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773315.752790:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773315.752801:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2311 00000100:00080000:2.0:1790773315.752813:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773315.752836:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773315.752878:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534648576/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773315.752923:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773315.752929:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773315.752932:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773315.752938:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2311 00000100:00080000:2.0:1790773315.752943:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773315.752950:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773315.752962:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534648576:192.168.204.150@tcp:250:kworker.0 00000008:00100000:2.0:1790773315.752962:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773315.752986:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534648704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773315.753000:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773315.753005:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773315.753007:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773315.753013:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2311 00000800:00000100:1.0:1790773315.753018:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773315.753018:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773315.753029:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773315.753036:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773315.753074:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534648832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773315.753078:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773315/real 1790773315] req@ffff9c4efbcc2a00 x1877759534648576/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773315.753105:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773315.753109:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534648576/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773315.753112:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773315.753115:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773315.753125:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2311 00000100:00080000:1.0:1790773315.753129:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773315.753131:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773315.753136:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773315.753143:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773315.753144:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534648576:192.168.204.150@tcp:250:kworker.0 00000008:00100000:2.0:1790773315.753156:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773315.753173:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534648704:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773315.753175:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534648960/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773315.753192:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773315.753196:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773315.753200:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773315.753204:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773315.753211:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773315/real 1790773315] req@ffff9c4efbcc1c00 x1877759534648704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773315.753212:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2311 00000100:00080000:2.0:1790773315.753220:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773315.753230:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773315.753236:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773315.753238:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534648704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773315.753253:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534649088/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773315.753273:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773315.753278:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773315.753282:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773315.753284:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1790773315.753288:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534648704:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773315.753290:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773315.753299:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2311 00000100:00100000:1.0:1790773315.753300:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534648832:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773315.753307:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773315.753316:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773315.753322:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773315.753325:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773315.753342:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534649216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773315.753346:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773315/real 1790773315] req@ffff9c4efbcc1f80 x1877759534648832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773315.753368:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773315.753373:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534648832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773315.753376:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773315.753380:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773315.753385:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773315.753390:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2311 00000100:00080000:1.0:1790773315.753391:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773315.753399:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534648832:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773315.753399:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773315.753409:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773315.753416:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534648960:192.168.204.150@tcp:8:kworker.0 00000008:00100000:2.0:1790773315.753417:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773315.753447:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534649344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773315.753455:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773315.753468:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773315/real 1790773315] req@ffff9c4efbcc1500 x1877759534648960/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773315.753468:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773315.753479:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773315.753484:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773315.753492:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534648960/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773315.753492:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2311 00000100:00080000:2.0:1790773315.753508:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773315.753517:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773315.753518:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773315.753525:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773315.753529:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534648960:192.168.204.150@tcp:8:kworker.0 00000008:00100000:2.0:1790773315.753529:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773315.753542:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534649088:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773315.753548:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534649472/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773315.753558:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773315.753562:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773315.753566:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773315.753572:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773315.753577:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773315/real 1790773315] req@ffff9c4efbcc1880 x1877759534649088/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773315.753579:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2311 00000100:00080000:2.0:1790773315.753588:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773315.753593:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534649088/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773315.753599:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773315.753607:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790773315.753610:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773315.753612:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773315.753618:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534649088:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773315.753624:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534649216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773315.753642:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534649600/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773315.753668:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773315.753678:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773315/real 1790773315] req@ffff9c4efbcc2300 x1877759534649216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773315.753715:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534649216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773315.753736:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773315.753741:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773315.753747:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534649216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773315.753773:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534649344:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773315.753791:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773315.753803:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773315/real 1790773315] req@ffff9c4efbcc0a80 x1877759534649344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773315.753816:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534649344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773315.753838:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773315.753841:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773315.753845:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534649344:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773315.753852:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534649472:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773315.753882:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773315.753895:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773315/real 1790773315] req@ffff9c4efbcc2680 x1877759534649472/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773315.753927:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534649472/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773315.753947:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773315.753950:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773315.753952:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534649472:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773315.753957:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534649600:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773315.753973:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773315.753982:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773315/real 1790773315] req@ffff9c4efbcc0700 x1877759534649600/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773315.753990:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534649600/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773315.754007:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773315.754009:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773315.754011:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534649600:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773315.816764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773315.816766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773315.816773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773315.816773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773315.880729:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773315.880764:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773315.880788:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773316.008724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773316.008824:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773316.072751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773316.136733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773316.200722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773316.200722:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773316.200723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773316.264718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773316.328743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773316.328744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773316.328745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773316.456716:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773316.456716:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773316.456719:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773316.456721:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773316.520754:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773316.520780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773316.520787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773316.520806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773316.520847:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773316.520848:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773316.648712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773316.712742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773316.776723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773316.840740:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773316.840741:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773316.840741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773316.840743:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773316.904740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773316.904757:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773316.904797:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773317.032726:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773317.032741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773317.096716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773317.160732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773317.224714:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773317.224717:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773317.224731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773317.288721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773317.352741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773317.352741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773317.352741:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773317.480743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773317.480744:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773317.480744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773317.480745:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773317.544711:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773317.544718:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773317.544724:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773317.544733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773317.544758:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773317.544761:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773317.672716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773317.736700:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773317.800712:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773317.864733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773317.864733:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773317.864734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773317.864735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773317.928745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773317.928755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773317.928836:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773318.056763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773318.056763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773318.120741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773318.184778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773318.248764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773318.248807:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773318.248808:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773318.312720:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773318.376767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773318.376801:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773318.376809:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773318.504750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773318.504751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773318.504761:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773318.504763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773318.568749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773318.568759:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773318.568782:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773318.568786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773318.568794:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773318.568820:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773318.696796:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773318.760792:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773318.824724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773318.888734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773318.888743:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773318.888756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773318.888804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773318.952716:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773318.952762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773318.952807:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773319.080705:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773319.080715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773319.144796:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773319.208751:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773319.272760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773319.272767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773319.272772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773319.336743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773319.400754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773319.400755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773319.400755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773319.528722:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773319.528723:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773319.528731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773319.528732:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773319.592734:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773319.592747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773319.592749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773319.592754:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773319.592776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773319.592777:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773319.720778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773319.784720:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773319.848712:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773319.912767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773319.912768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773319.912768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773319.912768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773319.976735:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773319.976742:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773319.976759:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773320.104711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773320.104714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773320.168789:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773320.232742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773320.296765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773320.296776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773320.296778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773320.360736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773320.424709:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773320.424717:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773320.424719:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773320.552713:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773320.552718:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773320.552719:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773320.552722:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773320.616721:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773320.616721:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773320.616726:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773320.616736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773320.616766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773320.616770:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773320.744801:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773320.808774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773320.872714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773320.872801:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773320.872807:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773320.872826:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773320.872834:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2316 00000100:00080000:0.0:1790773320.872843:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773320.872853:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773320.872888:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534649728/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773320.872916:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773320.872919:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773320.872920:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773320.872924:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2316 00000100:00080000:0.0:1790773320.872927:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773320.872931:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773320.872939:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773320.872950:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534649856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773320.872955:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534649728:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773320.872972:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773320.872976:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773320.872978:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773320.872984:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2316 00000100:00080000:0.0:1790773320.872989:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773320.873009:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773320.873016:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773320.873030:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534649984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773320.873043:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773320.873047:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773320.873051:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773320.873052:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534649856:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773320.873061:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2316 00000100:00080000:0.0:1790773320.873066:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773320.873071:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773320.873076:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773320.873088:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534650112/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773320.873092:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534649984:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773320.873105:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773320.873108:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773320.873110:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773320.873114:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2316 00000100:00080000:0.0:1790773320.873125:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773320.873130:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773320.873134:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773320.873134:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534650112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773320.873150:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534650240/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773320.873158:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773320.873162:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773320.873164:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773320.873170:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2316 00000100:00100000:1.0:1790773320.873170:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534650240:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773320.873180:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773320.873188:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773320.873193:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773320.873202:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534650368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773320.873219:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773320.873222:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773320.873224:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773320.873228:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2316 00000100:00080000:0.0:1790773320.873231:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773320.873237:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773320.873237:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534650368:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773320.873245:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773320.873256:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534650496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773320.873273:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773320.873276:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773320.873278:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773320.873282:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2316 00000100:00080000:0.0:1790773320.873287:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773320.873294:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773320.873301:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773320.873301:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534650496:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773320.873320:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534650624/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773320.873333:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773320.873337:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773320.873340:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773320.873344:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534650624:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773320.873348:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2316 00000100:00080000:0.0:1790773320.873356:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773320.873362:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773320.873369:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773320.873384:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534650752/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773320.873455:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534650752:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773320.936737:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773320.936738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773320.936738:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773320.936748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773321.000729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773321.000749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773321.000773:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773321.128703:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773321.128713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773321.192721:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773321.256741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773321.320769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773321.320769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773321.320794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773321.384720:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773321.448766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773321.448780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773321.448781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773321.576727:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773321.576736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773321.576806:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773321.576807:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773321.640753:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773321.640755:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773321.640760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773321.640837:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773321.640841:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773321.640867:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773321.768746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773321.832750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773321.896724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773321.960783:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773321.960783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773321.960784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773321.960802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773322.024800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773322.024806:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773322.024868:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773322.152737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773322.152741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773322.216750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773322.280757:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773322.344732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773322.344734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773322.344736:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773322.408718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773322.472732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773322.472742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773322.472743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773322.600752:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773322.600752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773322.600753:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773322.600753:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773322.664719:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773322.664720:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773322.664724:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773322.664737:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773322.664763:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773322.664766:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773322.792708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773322.856741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773322.920724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773322.984742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773322.984746:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773322.984748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773322.984748:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773323.048748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773323.048748:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773323.048847:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773323.176729:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773323.176736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773323.240716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773323.304726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773323.368776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773323.368776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773323.368777:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773323.432728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773323.496711:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773323.496726:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773323.496731:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773323.624751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773323.624799:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773323.624801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773323.624801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773323.688725:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773323.688769:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773323.688803:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773323.688844:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773323.688854:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773323.688899:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773323.816748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773323.880708:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773323.944810:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:2.0:1790773323.944922:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:2.0:1790773323.944971:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1790773323.944990:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773323.945048:0:2384:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:2.0:1790773323.945063:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773323.945075:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773323.945089:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773323.945098:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773323.945110:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773323.945122:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773323.945132:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:1.0:1790773323.945169:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773320/real 1790773323] req@ffff9c4ec7a7f480 x1877759534650752/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773323.945228:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534650752/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773323.945270:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773323.945281:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773323.945300:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534650752:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773323.945318:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773320/real 1790773323] req@ffff9c4ec7a7df80 x1877759534650624/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773323.945344:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534650624/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773323.945365:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773323.945373:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773323.945380:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534650624:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773323.945394:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773320/real 1790773323] req@ffff9c4ec7a7ed80 x1877759534650496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773323.945421:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534650496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773323.945468:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773323.945477:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773323.945484:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534650496:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773323.945514:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773320/real 1790773323] req@ffff9c4ec7a7d880 x1877759534650368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773323.945539:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534650368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773323.945570:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773323.945578:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773323.945584:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534650368:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773323.945599:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773320/real 1790773323] req@ffff9c4ec7a7c380 x1877759534650240/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773323.945634:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534650240/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773323.945692:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773323.945702:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773323.945710:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534650240:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773323.945725:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773320/real 1790773323] req@ffff9c4ec7a7ca80 x1877759534650112/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773323.945785:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534650112/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773323.945806:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773323.945814:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773323.945821:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534650112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773323.945843:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773320/real 1790773323] req@ffff9c4ec7a7fb80 x1877759534649856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773323.945873:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534649856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773323.945910:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773323.945931:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773323.945940:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534649856:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773323.945954:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773320/real 1790773323] req@ffff9c4ec7a7e300 x1877759534649984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773323.945981:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534649984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773323.946000:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773323.946008:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773323.946017:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534649984:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773323.946035:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773320/real 1790773323] req@ffff9c4ec7a7c700 x1877759534649728/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773323.946068:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534649728/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773323.946088:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773323.946097:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773323.946104:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534649728:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773324.008740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773324.008740:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773324.008741:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773324.008807:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773324.072780:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773324.072854:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773324.072946:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773324.200795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773324.200810:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773324.264811:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773324.328835:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773324.392805:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773324.392839:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773324.392855:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773324.456812:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773324.520813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773324.520821:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773324.520836:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773324.648845:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773324.648857:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773324.648897:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773324.648901:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773324.712818:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773324.712825:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773324.712883:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773324.712885:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773324.712918:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773324.712936:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773324.840810:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773324.904836:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773324.968944:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773325.032776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773325.032808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773325.032818:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773325.032843:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773325.096860:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773325.096870:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773325.096988:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773325.224750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773325.224750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773325.288864:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773325.352825:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773325.416786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773325.416804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773325.416806:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773325.480771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773325.544746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773325.544788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773325.544855:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773325.672849:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773325.672868:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773325.672869:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773325.672937:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773325.736747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773325.736792:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773325.736792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773325.736803:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773325.736877:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773325.736886:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773325.864736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773325.928779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773325.992851:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773325.992868:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773325.992878:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773325.992896:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773325.992898:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2321 00000100:00080000:0.0:1790773325.992926:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773325.992948:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773325.993000:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534650880/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773325.993054:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773325.993060:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773325.993063:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773325.993069:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2321 00000100:00080000:0.0:1790773325.993075:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773325.993081:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773325.993093:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773325.993115:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534651008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773325.993131:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773325.993140:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773325.993148:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773325.993148:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534650880:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773325.993169:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2321 00000100:00080000:0.0:1790773325.993180:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773325.993191:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773325.993202:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773325.993229:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534651136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773325.993252:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773325.993261:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773325.993266:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773325.993275:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2321 00000100:00080000:0.0:1790773325.993308:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773325.993329:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773325.993335:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773325.993352:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773325.993392:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534651264/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773325.993415:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773325.993423:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773325.993434:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773325/real 1790773325] req@ffff9c4ec7a7d180 x1877759534650880/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773325.993435:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773325.993457:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2321 00000100:00080000:0.0:1790773325.993472:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773325.993494:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773325.993506:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773325.993511:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534650880/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773325.993542:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534651392/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773325.993561:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773325.993567:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773325.993586:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773325.993587:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773325.993600:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773325.993609:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534650880:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773325.993621:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2321 00000100:00080000:0.0:1790773325.993636:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773325.993705:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773325.993744:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773325.993780:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534651520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773325.993794:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534651008:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773325.993835:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773325.993845:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773325.993852:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773325.993867:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2321 00000100:00080000:0.0:1790773325.993883:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773325.993895:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773325.993956:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773325.993967:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773325/real 1790773325] req@ffff9c4ec7a7c700 x1877759534651008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773325.993976:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773325.994031:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534651008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773325.994040:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534651648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773325.994114:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773325.994120:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773325.994125:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773325.994129:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773325.994142:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2321 00000100:00080000:1.0:1790773325.994155:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773325.994159:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773325.994171:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534651008:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773325.994183:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773325.994200:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773325.994205:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534651136:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773325.994230:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534651776/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773325.994255:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773325.994261:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773325.994267:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773325.994277:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2321 00000100:00080000:0.0:1790773325.994286:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773325.994295:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773325.994307:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773325.994318:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773325.994342:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773325/real 1790773325] req@ffff9c4ec7a7e300 x1877759534651136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773325.994352:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534651904/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773325.994388:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534651136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773325.994444:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773325.994456:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773325.994467:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534651136:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773325.994486:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534651264:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773325.994536:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773325.994559:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773325/real 1790773325] req@ffff9c4ec7a7fb80 x1877759534651264/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773325.994584:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534651264/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773325.994603:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773325.994610:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773325.994618:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534651264:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773325.994630:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534651392:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773325.994747:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773325.994775:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773325/real 1790773325] req@ffff9c4ec7a7ca80 x1877759534651392/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773325.994799:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534651392/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773325.994816:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773325.994823:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773325.994830:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534651392:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773325.994871:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534651520:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773325.994914:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773325.994937:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773325/real 1790773325] req@ffff9c4ec7a7c380 x1877759534651520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773325.994959:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534651520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773325.994986:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773325.994993:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773325.994999:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534651520:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773325.995013:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534651648:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773325.995054:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773325.995079:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773325/real 1790773325] req@ffff9c4ec7a7d880 x1877759534651648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773325.995103:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534651648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773325.995131:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773325.995138:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773325.995145:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534651648:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773325.995157:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534651776:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773325.995198:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773325.995220:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773325/real 1790773325] req@ffff9c4ec7a7ed80 x1877759534651776/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773325.995245:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534651776/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773325.995265:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773325.995273:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773325.995280:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534651776:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773325.995293:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534651904:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773325.995362:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773325.995387:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773325/real 1790773325] req@ffff9c4ec7a7df80 x1877759534651904/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773325.995411:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534651904/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773325.995431:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773325.995439:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773325.995447:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534651904:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773326.056823:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773326.056823:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773326.056842:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773326.056857:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773326.120839:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773326.120879:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773326.121063:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773326.248805:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773326.248818:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773326.312764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773326.376749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773326.440731:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773326.440754:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773326.440754:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773326.504723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773326.568758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773326.568782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773326.568792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773326.696796:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773326.696803:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773326.696818:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773326.696818:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773326.760731:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773326.760732:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773326.760763:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773326.760768:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773326.760787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773326.760787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773326.888734:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773326.952730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773327.016808:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773327.080750:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773327.080774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773327.080781:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773327.080812:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773327.144738:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773327.144738:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773327.144884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773327.272760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773327.272791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773327.336824:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773327.400770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773327.464825:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773327.464825:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773327.464834:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773327.528734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773327.592762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773327.592786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773327.592791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773327.720752:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773327.720760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773327.720764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773327.720764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773327.784767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773327.784767:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773327.784767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773327.784780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773327.784829:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773327.784833:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773327.912749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773327.976727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773328.040715:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773328.104766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773328.104775:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773328.104792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773328.104799:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773328.168785:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773328.168787:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773328.168886:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773328.296738:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773328.296742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773328.360767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773328.424819:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773328.488749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773328.488761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773328.488764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773328.552736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773328.616797:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773328.616797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773328.616798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773328.744764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773328.744774:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773328.744776:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773328.744776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773328.808746:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773328.808746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773328.808747:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773328.808765:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773328.808790:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773328.808802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773328.936730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773329.000739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773329.064768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773329.128746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773329.128746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773329.128752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773329.128755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773329.192760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773329.192760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773329.192880:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773329.320765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773329.320766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773329.384779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773329.448744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773329.512807:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773329.512812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773329.512816:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773329.576761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773329.640742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773329.640746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773329.640749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773329.768804:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773329.768804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773329.768824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773329.768825:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773329.832715:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773329.832746:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773329.832757:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773329.832788:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773329.832801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773329.832872:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773329.960787:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773330.024756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773330.088733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773330.152745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773330.152745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773330.152753:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773330.152753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773330.216716:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773330.216726:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773330.216778:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773330.344718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773330.344729:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773330.408772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773330.472747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773330.536750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773330.536751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773330.536756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773330.600777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773330.664779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773330.664779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773330.664780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773330.792740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773330.792741:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773330.792749:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773330.792750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773330.856736:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773330.856743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773330.856768:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773330.856790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773330.856793:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773330.856815:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773330.984708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773331.048765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773331.112821:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773331.112821:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773331.112833:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773331.112839:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773331.112852:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2327 00000100:00080000:0.0:1790773331.112867:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773331.112885:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773331.112936:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534652032/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773331.112974:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773331.112982:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773331.112985:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773331.112993:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2327 00000100:00080000:0.0:1790773331.113000:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773331.113009:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773331.113024:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773331.113032:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534652032:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773331.113047:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534652160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773331.113065:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773331.113071:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773331.113074:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773331.113080:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2327 00000100:00080000:0.0:1790773331.113099:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773331.113107:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773331.113113:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773331.113116:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773331.113134:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534652288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773331.113150:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773331.113156:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773331.113159:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773331.113166:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2327 00000100:00100000:1.0:1790773331.113167:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773331/real 1790773331] req@ffff9c4ec7a7f480 x1877759534652032/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773331.113172:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773331.113180:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773331.113188:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773331.113195:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534652032/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773331.113205:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534652416/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773331.113212:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773331.113219:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773331.113221:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773331.113227:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773331.113227:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534652032:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773331.113231:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773331.113238:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2327 00000100:00080000:0.0:1790773331.113244:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773331.113251:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773331.113258:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773331.113258:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534652160:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773331.113273:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534652544/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773331.113290:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773331.113297:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:1.0:1790773331.113299:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773331.113301:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773331.113308:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2327 00000100:00080000:0.0:1790773331.113314:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773331.113318:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773331/real 1790773331] req@ffff9c4ec7a7df80 x1877759534652160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773331.113322:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773331.113330:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773331.113340:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534652160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773331.113346:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534652672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773331.113379:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773331.113385:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773331.113385:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773331.113389:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773331.113393:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773331.113398:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2327 00000100:00100000:1.0:1790773331.113401:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534652160:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773331.113404:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773331.113414:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773331.113414:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534652288:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773331.113423:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773331.113439:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534652800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773331.113450:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773331.113460:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773331.113482:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1790773331.113485:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773331/real 1790773331] req@ffff9c4ec7a7ed80 x1877759534652288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773331.113486:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773331.113493:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2327 00000100:00080000:0.0:1790773331.113500:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773331.113505:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534652288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773331.113507:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773331.113515:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773331.113522:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773331.113529:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773331.113531:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534652928/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773331.113536:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534652288:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773331.113546:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534652416:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773331.113549:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773331.113555:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773331.113559:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773331.113566:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2327 00000100:00080000:0.0:1790773331.113588:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773331.113592:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773331.113597:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773331.113606:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773331.113694:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773331/real 1790773331] req@ffff9c4ec7a7d880 x1877759534652416/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773331.113743:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534653056/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773331.113754:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534652416/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773331.113774:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773331.113782:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773331.113790:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534652416:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773331.113837:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534652544:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773331.113871:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773331.113886:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773331/real 1790773331] req@ffff9c4ec7a7c380 x1877759534652544/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773331.113899:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534652544/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773331.113911:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773331.113915:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773331.113919:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534652544:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773331.113926:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534652672:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773331.113945:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773331.113955:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773331/real 1790773331] req@ffff9c4ec7a7ca80 x1877759534652672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773331.113968:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534652672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773331.113992:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773331.113996:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773331.114000:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534652672:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773331.114007:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534652800:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773331.114023:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773331.114033:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773331/real 1790773331] req@ffff9c4ec7a7fb80 x1877759534652800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773331.114045:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534652800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773331.114055:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773331.114058:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773331.114062:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534652800:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773331.114068:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534652928:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773331.114084:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773331.114094:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773331/real 1790773331] req@ffff9c4ec7a7e300 x1877759534652928/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773331.114107:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534652928/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773331.114118:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773331.114122:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773331.114126:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534652928:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773331.114133:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534653056:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773331.114149:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773331.114158:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773331/real 1790773331] req@ffff9c4ec7a7c700 x1877759534653056/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773331.114170:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534653056/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773331.114180:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773331.114184:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773331.114187:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534653056:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773331.176731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773331.176759:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773331.176764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773331.176798:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773331.240780:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773331.240853:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773331.240861:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773331.368769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773331.368798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773331.432786:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773331.496788:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773331.560773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773331.560790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773331.560800:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773331.624769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773331.688766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773331.688814:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773331.688827:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773331.816778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773331.816787:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773331.816791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773331.816801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773331.880803:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773331.880831:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773331.880835:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773331.880856:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773331.880879:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773331.880886:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773332.008785:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773332.072805:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773332.136763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773332.200749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773332.200765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773332.200766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773332.200774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773332.264757:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773332.264787:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773332.264793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773332.392716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773332.392726:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773332.456849:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773332.520750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773332.584754:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773332.584756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773332.584759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773332.648738:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773332.712751:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773332.712772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773332.712786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773332.840755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773332.840764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773332.840769:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773332.840771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773332.904754:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773332.904754:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773332.904801:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773332.904803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773332.904813:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773332.904823:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773333.032766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773333.096731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773333.160739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773333.224745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773333.224745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773333.224754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773333.224755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773333.288772:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773333.288815:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773333.288844:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773333.416748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773333.416748:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773333.480721:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773333.544744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773333.608735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773333.608762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773333.608762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773333.672729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773333.736756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773333.736770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773333.736770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773333.864763:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773333.864763:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773333.864763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773333.864766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773333.928714:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773333.928719:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773333.928744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773333.928749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773333.928762:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773333.928767:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773334.056739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773334.120763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773334.184732:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773334.248734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773334.248734:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773334.248735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773334.248735:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773334.312727:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773334.312737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773334.312775:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773334.440702:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773334.440709:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773334.504771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773334.568716:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773334.632717:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773334.632717:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773334.632723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773334.696722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773334.760807:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773334.760807:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773334.760807:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773334.888748:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773334.888748:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773334.888750:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773334.888755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773334.952736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773334.952755:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773334.952761:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773334.952801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773334.952802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773334.952813:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773335.080718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773335.144708:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773335.208718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773335.272729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773335.272742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773335.272743:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773335.272743:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773335.336734:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773335.336772:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773335.336778:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773335.464726:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773335.464733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773335.528722:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773335.592754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773335.656767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773335.656767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773335.656767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773335.720758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773335.784745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773335.784746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773335.784748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773335.912749:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773335.912749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773335.912750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773335.912751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773335.976719:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773335.976720:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773335.976734:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773335.976740:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773335.976763:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773335.976768:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773336.104710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773336.168747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773336.232763:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773336.232765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773336.232776:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773336.232782:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773336.232809:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2332 00000100:00080000:0.0:1790773336.232818:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773336.232828:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773336.232861:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534653184/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773336.232888:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773336.232891:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773336.232893:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773336.232897:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2332 00000100:00080000:0.0:1790773336.232899:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773336.232903:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773336.232911:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773336.232922:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534653312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773336.232931:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773336.232932:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534653184:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773336.232937:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773336.232941:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773336.232945:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2332 00000100:00080000:0.0:1790773336.232948:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773336.232951:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773336.232954:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773336.232962:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534653440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773336.232969:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773336.232972:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773336.232973:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773336.232976:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2332 00000100:00080000:0.0:1790773336.232993:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773336.232997:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773336.233001:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:1.0:1790773336.233005:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773336.233013:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534653568/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773336.233020:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773336.233023:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773336.233025:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773336.233029:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2332 00000100:00080000:0.0:1790773336.233031:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773336.233034:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773336.233037:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773336.233045:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534653696/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773336.233051:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773336.233054:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773336.233056:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773336.233058:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2332 00000100:00080000:0.0:1790773336.233060:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773336.233063:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773336.233066:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773336.233073:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773336/real 1790773336] req@ffff9c4ec7a7d180 x1877759534653184/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773336.233074:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534653824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773336.233106:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773336.233111:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1790773336.233112:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534653184/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773336.233113:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773336.233122:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2332 00000100:00080000:0.0:1790773336.233128:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773336.233133:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773336.233136:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773336.233143:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534653952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773336.233150:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773336.233150:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773336.233154:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773336.233158:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773336.233162:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773336.233163:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2332 00000100:00080000:0.0:1790773336.233168:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773336.233174:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773336.233174:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534653184:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773336.233180:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773336.233190:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534654080/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773336.233196:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773336.233198:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773336.233200:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773336.233203:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2332 00000100:00080000:0.0:1790773336.233207:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773336.233210:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534653312:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773336.233213:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773336.233219:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773336.233227:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534654208/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773336.233249:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773336.233266:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773336/real 1790773336] req@ffff9c4ec7a7c700 x1877759534653312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773336.233281:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534653312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773336.233300:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773336.233304:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773336.233308:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534653312:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773336.233316:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534653440:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773336.233338:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773336.233350:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773336/real 1790773336] req@ffff9c4ec7a7e300 x1877759534653440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773336.233363:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534653440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773336.233373:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773336.233378:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773336.233382:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534653440:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773336.233390:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534653568:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773336.233411:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773336.233423:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773336/real 1790773336] req@ffff9c4ec7a7fb80 x1877759534653568/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773336.233437:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534653568/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773336.233448:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773336.233452:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773336.233456:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534653568:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773336.233463:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534653696:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773336.233483:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773336.233494:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773336/real 1790773336] req@ffff9c4ec7a7ca80 x1877759534653696/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773336.233507:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534653696/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773336.233533:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773336.233537:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773336.233541:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534653696:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773336.233548:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534653824:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773336.233571:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773336.233583:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773336/real 1790773336] req@ffff9c4ec7a7c380 x1877759534653824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773336.233597:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534653824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773336.233615:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773336.233619:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773336.233623:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534653824:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773336.233630:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534653952:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773336.233673:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773336.233686:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773336/real 1790773336] req@ffff9c4ec7a7d880 x1877759534653952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773336.233701:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534653952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773336.233730:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773336.233735:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773336.233741:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534653952:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773336.233750:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534654080:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773336.233782:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773336.233813:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773336/real 1790773336] req@ffff9c4ec7a7ed80 x1877759534654080/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773336.233825:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534654080/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773336.233844:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773336.233848:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773336.233852:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534654080:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773336.233876:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534654208:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773336.233898:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773336.233910:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773336/real 1790773336] req@ffff9c4ec7a7df80 x1877759534654208/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773336.233923:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534654208/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773336.233934:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773336.233938:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773336.233942:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534654208:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773336.296748:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773336.296751:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773336.296755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773336.296756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773336.360723:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773336.360730:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773336.360813:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773336.488732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773336.488737:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773336.552728:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773336.616712:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773336.680724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773336.680727:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773336.680728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773336.744735:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773336.808731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773336.808734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773336.808735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773336.936764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773336.936765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773336.936765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773336.936791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773337.000757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773337.000757:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773337.000757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773337.000761:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773337.000792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773337.000793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773337.128750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773337.192751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773337.256734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773337.320739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773337.320740:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773337.320741:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773337.320758:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773337.384767:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773337.384775:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773337.384806:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773337.512737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773337.512741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773337.576740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773337.640733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773337.704752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773337.704753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773337.704754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773337.768745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773337.832775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773337.832791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773337.832791:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773337.960793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773337.960793:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773337.960799:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773337.960818:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773338.024745:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773338.024770:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773338.024782:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773338.024787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773338.024819:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773338.024824:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773338.152721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773338.216783:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773338.280789:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773338.344731:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773338.344733:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773338.344738:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773338.344741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773338.408804:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773338.408875:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773338.408884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773338.536767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773338.536776:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773338.600760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773338.664723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773338.728741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773338.728755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773338.728779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773338.792716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773338.856813:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773338.856821:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773338.856824:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773338.984752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773338.984797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773338.984801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773338.984806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773339.048728:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773339.048729:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773339.048740:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773339.048778:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773339.048795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773339.048806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773339.176730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773339.240743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773339.304771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773339.368759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773339.368759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773339.368774:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773339.368776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773339.432787:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773339.432835:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773339.432865:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773339.560727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773339.560761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773339.624791:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773339.688746:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773339.752718:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773339.752760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773339.752778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773339.816741:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773339.880739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773339.880755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773339.880777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773340.008754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773340.008772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773340.008773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773340.008774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773340.072737:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773340.072742:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773340.072745:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773340.072755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773340.072769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773340.072784:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773340.200718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773340.264787:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773340.328748:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773340.392715:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773340.392761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773340.392764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773340.392775:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773340.456733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773340.456733:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773340.456840:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773340.584753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773340.584843:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773340.648748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773340.712752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773340.776696:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773340.776705:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773340.776722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773340.840743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773340.904723:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773340.904723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773340.904744:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773341.032733:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773341.032733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773341.032733:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773341.032740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773341.096765:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773341.096774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773341.096781:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773341.096782:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773341.096824:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773341.096832:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773341.224719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773341.288703:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773341.352726:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773341.352735:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773341.352739:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773341.352750:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2337 00000100:00100000:1.0:1790773341.352758:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773341.352762:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773341.352780:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773341.352825:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534654336/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773341.352848:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773341.352852:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773341.352855:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773341.352860:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2337 00000100:00080000:0.0:1790773341.352865:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773341.352874:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773341.352885:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773341.352889:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534654336:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773341.352909:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534654464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773341.352924:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773341.352927:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773341.352929:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773341.352932:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2337 00000100:00080000:0.0:1790773341.352935:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773341.352941:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773341.352945:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773341.352954:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534654592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773341.352961:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773341.352964:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773341.352966:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773341.352969:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2337 00000100:00080000:0.0:1790773341.352972:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773341.352975:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773341.352979:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:1.0:1790773341.352980:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773341.352995:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534654720/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773341.353005:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773341.353008:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773341.353010:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773341.353014:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2337 00000100:00080000:0.0:1790773341.353018:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773341.353024:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773341.353031:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773341.353037:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773341/real 1790773341] req@ffff9c4ec7a7f480 x1877759534654336/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773341.353044:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534654848/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773341.353053:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773341.353057:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773341.353066:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773341.353076:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2337 00000100:00080000:0.0:1790773341.353080:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773341.353083:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534654336/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773341.353094:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773341.353098:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790773341.353104:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773341.353104:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773341.353111:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534654336:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773341.353121:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534654976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773341.353130:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773341.353133:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773341.353134:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773341.353138:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2337 00000100:00100000:1.0:1790773341.353143:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534654464:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773341.353144:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773341.353159:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773341.353167:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773341.353186:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773341.353187:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534655104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773341.353200:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773341/real 1790773341] req@ffff9c4ec7a7df80 x1877759534654464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773341.353247:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773341.353251:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534654464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773341.353252:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773341.353258:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773341.353274:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2337 00000100:00080000:0.0:1790773341.353277:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773341.353286:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773341.353292:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773341.353306:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773341.353307:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534655232/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773341.353324:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773341.353326:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773341.353334:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773341.353339:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773341.353339:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534654464:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773341.353360:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2337 00000100:00100000:1.0:1790773341.353363:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534654592:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773341.353368:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773341.353379:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773341.353384:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773341.353397:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534655360/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773341.353404:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773341.353442:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773341/real 1790773341] req@ffff9c4ec7a7ed80 x1877759534654592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773341.353460:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534654592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773341.353467:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773341.353471:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773341.353473:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534654592:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773341.353481:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534654720:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773341.353508:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773341.353521:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773341/real 1790773341] req@ffff9c4ec7a7d880 x1877759534654720/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773341.353535:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534654720/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773341.353544:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773341.353547:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773341.353549:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534654720:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773341.353555:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534654848:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773341.353576:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773341.353589:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773341/real 1790773341] req@ffff9c4ec7a7c380 x1877759534654848/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773341.353602:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534654848/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773341.353621:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773341.353625:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773341.353628:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534654848:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773341.353633:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534654976:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773341.353689:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773341.353704:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773341/real 1790773341] req@ffff9c4ec7a7ca80 x1877759534654976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773341.353713:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534654976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773341.353725:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773341.353728:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773341.353731:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534654976:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773341.353748:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534655104:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773341.353765:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773341.353773:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773341/real 1790773341] req@ffff9c4ec7a7fb80 x1877759534655104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773341.353785:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534655104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773341.353794:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773341.353798:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773341.353802:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534655104:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773341.353810:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534655232:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773341.353832:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773341.353842:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773341/real 1790773341] req@ffff9c4ec7a7e300 x1877759534655232/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773341.353850:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534655232/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773341.353858:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773341.353861:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773341.353864:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534655232:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773341.353868:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534655360:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773341.353881:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773341.353888:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773341/real 1790773341] req@ffff9c4ec7a7c700 x1877759534655360/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773341.353901:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534655360/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773341.353911:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773341.353915:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773341.353919:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534655360:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773341.416777:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773341.416779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773341.416781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773341.416781:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773341.480728:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773341.480755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773341.480884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773341.608799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773341.608799:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773341.672788:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773341.736841:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773341.800709:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773341.800709:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773341.800766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773341.864736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773341.928730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773341.928763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773341.928763:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773342.056797:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773342.056797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773342.056805:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773342.056807:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773342.120746:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773342.120747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773342.120771:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773342.120772:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773342.120811:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773342.120834:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773342.248724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773342.312721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773342.376724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773342.440763:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773342.440763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773342.440763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773342.440764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773342.504720:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773342.504730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773342.504926:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773342.632732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773342.632772:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773342.696768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773342.760798:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773342.824717:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773342.824720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773342.824753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773342.888723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773342.952725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773342.952729:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773342.952766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773343.080759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773343.080771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773343.080801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773343.080819:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773343.144750:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773343.144754:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773343.144754:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773343.144758:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773343.144799:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773343.144809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773343.272747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773343.336728:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773343.400772:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773343.464756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773343.464756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773343.464757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773343.464769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773343.528704:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773343.528764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773343.528827:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773343.656749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773343.656749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773343.720753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773343.784753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773343.848698:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773343.848703:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773343.848717:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773343.912722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773343.976737:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773343.976737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773343.976737:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773344.104731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773344.104731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773344.104732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773344.104735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773344.168719:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773344.168724:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773344.168746:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773344.168747:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773344.168763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773344.168765:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773344.296706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773344.360737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773344.424737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773344.488741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773344.488774:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773344.488799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773344.488803:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773344.552753:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773344.552753:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773344.552846:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773344.680703:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773344.680712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773344.744743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773344.808738:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773344.872705:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773344.872706:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773344.872715:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773344.936740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773345.000747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773345.000747:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773345.000747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773345.128753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773345.128766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773345.128806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773345.128835:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773345.192793:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773345.192813:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773345.192840:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773345.192844:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773345.192885:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773345.192891:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773345.320721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773345.384723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773345.448740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773345.512743:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773345.512743:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773345.512743:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773345.512743:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773345.576723:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773345.576748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773345.576774:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773345.704720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773345.704728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773345.768706:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773345.832730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773345.896719:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773345.896721:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773345.896722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773345.960710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773346.024744:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773346.024744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773346.024744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773346.152747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773346.152747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773346.152754:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773346.152791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773346.216782:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773346.216782:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773346.216782:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773346.216789:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773346.216831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773346.216838:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773346.344736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773346.408762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773346.472719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773346.472738:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773346.472746:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773346.472751:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773346.472760:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2342 00000100:00080000:0.0:1790773346.472771:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773346.472780:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773346.472814:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534655488/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773346.472839:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773346.472842:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773346.472843:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773346.472846:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2342 00000100:00080000:0.0:1790773346.472849:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773346.472852:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773346.472861:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773346.472872:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534655616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773346.472899:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773346.472906:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534655488:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773346.472910:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773346.472918:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773346.472927:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2342 00000100:00080000:0.0:1790773346.472934:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773346.472960:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773346.472968:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773346.472985:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534655744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773346.472998:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773346.473003:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773346.473006:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773346.473014:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773346.473020:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2342 00000100:00080000:0.0:1790773346.473025:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773346.473033:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773346.473041:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773346.473054:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534655872/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773346.473062:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773346.473065:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773346.473067:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773346.473072:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2342 00000100:00080000:0.0:1790773346.473091:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773346.473091:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773346/real 1790773346] req@ffff9c4ec7a7d180 x1877759534655488/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773346.473109:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773346.473122:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773346.473134:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534655488/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773346.473135:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534656000/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773346.473227:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773346.473229:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773346.473233:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773346.473237:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773346.473244:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773346.473249:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2342 00000100:00080000:0.0:1790773346.473255:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773346.473260:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534655488:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773346.473267:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773346.473278:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773346.473326:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534656128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773346.473333:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534655616:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773346.473355:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773346.473376:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773346.473378:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773346.473383:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2342 00000100:00080000:0.0:1790773346.473388:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773346.473388:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773346.473399:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773346.473404:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773346.473419:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534656256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773346.473421:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773346/real 1790773346] req@ffff9c4ec7a7c700 x1877759534655616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773346.473447:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773346.473466:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773346.473468:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773346.473473:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2342 00000100:00080000:0.0:1790773346.473477:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773346.473480:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534655616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773346.473485:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773346.473496:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773346.473513:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534656384/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773346.473519:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773346.473523:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773346.473524:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773346.473529:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2342 00000100:00080000:1.0:1790773346.473531:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773346.473532:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773346.473546:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773346.473551:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773346.473561:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773346.473561:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534655616:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773346.473580:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534656512/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773346.473587:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534655744:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773346.473629:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773346.473640:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773346/real 1790773346] req@ffff9c4ec7a7e300 x1877759534655744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773346.473649:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534655744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773346.473676:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773346.473679:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773346.473682:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534655744:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773346.473686:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534655872:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773346.473701:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773346.473708:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773346/real 1790773346] req@ffff9c4ec7a7fb80 x1877759534655872/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773346.473716:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534655872/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773346.473723:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773346.473725:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773346.473727:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534655872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773346.473732:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534656000:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773346.473743:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773346.473749:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773346/real 1790773346] req@ffff9c4ec7a7ca80 x1877759534656000/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773346.473758:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534656000/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773346.473765:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773346.473767:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773346.473769:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534656000:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773346.473784:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534656128:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773346.473796:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773346.473803:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773346/real 1790773346] req@ffff9c4ec7a7c380 x1877759534656128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773346.473811:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534656128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773346.473823:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773346.473825:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773346.473827:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534656128:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773346.473832:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534656256:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773346.473843:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773346.473849:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773346/real 1790773346] req@ffff9c4ec7a7d880 x1877759534656256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773346.473857:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534656256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773346.473868:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773346.473870:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773346.473873:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534656256:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773346.473877:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534656384:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773346.473886:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773346.473897:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773346/real 1790773346] req@ffff9c4ec7a7ed80 x1877759534656384/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773346.473905:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534656384/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773346.473913:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773346.473915:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773346.473917:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534656384:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773346.473922:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534656512:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773346.473933:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773346.473939:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773346/real 1790773346] req@ffff9c4ec7a7df80 x1877759534656512/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773346.473947:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534656512/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773346.473965:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773346.473967:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773346.473970:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534656512:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773346.536734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773346.536734:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773346.536772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773346.536780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773346.600755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773346.600784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773346.600919:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773346.728715:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773346.728715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773346.792746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773346.856739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773346.920763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773346.920763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773346.920788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773346.984705:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773347.048719:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773347.048732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773347.048751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773347.176759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773347.176759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773347.176760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773347.176762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773347.240737:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773347.240782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773347.240784:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773347.240797:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773347.240830:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773347.240859:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773347.368758:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773347.432750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773347.496716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773347.560736:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773347.560737:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773347.560738:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773347.560777:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773347.624739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773347.624759:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773347.624832:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773347.752768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773347.752793:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773347.816779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773347.880749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773347.944749:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773347.944790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773347.944790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773348.008777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773348.072764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773348.072796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773348.072801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773348.200727:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773348.200736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773348.200763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773348.200773:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773348.264752:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773348.264815:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773348.264815:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773348.264815:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773348.264821:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773348.264872:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773348.392712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773348.456755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773348.520746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773348.584783:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773348.584814:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773348.584814:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773348.584828:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773348.648804:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773348.648823:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773348.648926:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773348.776720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773348.776720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773348.840778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773348.904765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773348.968756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773348.968756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773348.968805:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773349.032732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773349.096713:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773349.096736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773349.096750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773349.224744:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773349.224758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773349.224762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773349.224779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773349.288756:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773349.288784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773349.288830:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773349.288843:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773349.288844:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773349.288857:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773349.416769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773349.480770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773349.544763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773349.608722:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773349.608724:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773349.608800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773349.608800:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773349.672752:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773349.672790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773349.672901:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773349.800742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773349.800742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773349.864802:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773349.928932:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773349.992773:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773349.992776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773349.992790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773350.056756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773350.120817:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773350.120821:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773350.120821:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773350.248748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773350.248751:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773350.248785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773350.248786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773350.312722:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773350.312732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773350.312747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773350.312755:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773350.312780:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773350.312786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773350.440740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773350.504753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773350.568724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773350.632736:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773350.632744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773350.632772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773350.632778:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773350.696769:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773350.696782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773350.696859:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773350.824797:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773350.824798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773350.888792:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773350.952760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773351.016810:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773351.016810:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773351.016821:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773351.080756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773351.144808:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773351.144808:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773351.144819:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773351.272807:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773351.272807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773351.272807:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773351.272822:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773351.336750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773351.336759:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773351.336762:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773351.336764:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773351.336798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773351.336805:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773351.464733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773351.528766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773351.592736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773351.592783:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773351.592790:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773351.592794:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773351.592811:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2347 00000100:00080000:3.0:1790773351.592821:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773351.592832:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773351.592867:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534656640/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773351.592893:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773351.592897:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773351.592899:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773351.592903:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2347 00000100:00080000:3.0:1790773351.592915:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773351.592920:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773351.592930:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773351.592932:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534656640:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773351.592949:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534656768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773351.592960:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773351.592964:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773351.592967:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773351.592971:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2347 00000100:00080000:3.0:1790773351.592977:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773351.592981:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773351.592985:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773351.592997:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534656896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773351.593002:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773351.593014:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773351.593018:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773351.593020:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773351.593025:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2347 00000100:00080000:3.0:1790773351.593028:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773351.593033:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773351.593038:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773351.593049:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773351/real 1790773351] req@ffff9c4ef73c1c00 x1877759534656640/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773351.593052:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534657024/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773351.593083:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773351.593086:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773351.593090:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773351.593091:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534656640/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773351.593097:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2347 00000100:00080000:3.0:1790773351.593103:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773351.593111:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773351.593113:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773351.593116:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:1.0:1790773351.593122:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773351.593128:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534656640:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773351.593131:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534657152/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773351.593142:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773351.593146:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773351.593147:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773351.593151:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2347 00000100:00080000:3.0:1790773351.593156:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773351.593158:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534656768:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773351.593167:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773351.593175:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773351.593187:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773351.593196:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534657280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773351.593206:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773351/real 1790773351] req@ffff9c4ef73c2300 x1877759534656768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773351.593235:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773351.593246:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773351.593251:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773351.593262:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2347 00000100:00080000:3.0:1790773351.593269:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773351.593275:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534656768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773351.593276:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773351.593285:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773351.593304:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534657408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773351.593309:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773351.593316:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773351.593318:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773351.593323:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1790773351.593327:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534656768:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773351.593328:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773351.593339:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2347 00000100:00100000:1.0:1790773351.593344:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534656896:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773351.593345:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773351.593354:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773351.593360:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773351.593373:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534657536/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773351.593384:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773351.593395:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773351.593401:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773351.593402:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773351/real 1790773351] req@ffff9c4ef73c0e00 x1877759534656896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773351.593403:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773351.593421:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2347 00000100:00080000:3.0:1790773351.593427:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773351.593431:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534656896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773351.593434:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773351.593441:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773351.593457:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790773351.593457:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534657664/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773351.593471:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773351.593480:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534656896:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773351.593488:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534657024:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773351.593512:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773351.593523:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773351/real 1790773351] req@ffff9c4ef73c0a80 x1877759534657024/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773351.593533:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534657024/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773351.593548:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773351.593551:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773351.593555:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534657024:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773351.593560:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534657152:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773351.593573:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773351.593581:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773351/real 1790773351] req@ffff9c4ef73c3800 x1877759534657152/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773351.593590:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534657152/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773351.593598:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773351.593601:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773351.593604:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534657152:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773351.593621:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534657280:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773351.593636:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773351.593644:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773351/real 1790773351] req@ffff9c4ef73c2d80 x1877759534657280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773351.593679:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534657280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773351.593694:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773351.593697:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773351.593701:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534657280:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773351.593707:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534657408:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773351.593723:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773351.593732:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773351/real 1790773351] req@ffff9c4ef73c2a00 x1877759534657408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773351.593762:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534657408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773351.593773:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773351.593777:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773351.593782:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534657408:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773351.593789:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534657536:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773351.593810:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773351.593821:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773351/real 1790773351] req@ffff9c4ef73c0700 x1877759534657536/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773351.593835:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534657536/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773351.593848:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773351.593852:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773351.593857:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534657536:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773351.593863:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534657664:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773351.593881:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773351.593893:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773351/real 1790773351] req@ffff9c4ef73c1500 x1877759534657664/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773351.593908:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534657664/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773351.593920:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773351.593924:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773351.593928:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534657664:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773351.656739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773351.656744:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773351.656746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773351.656746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773351.720739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773351.720747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773351.720751:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773351.848761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773351.848785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773351.912743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773351.976755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773352.040745:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773352.040751:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773352.040789:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773352.104727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773352.168765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773352.168770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773352.168774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773352.296730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773352.296730:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773352.296733:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773352.296743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773352.360717:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773352.360717:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773352.360728:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773352.360746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773352.360762:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773352.360767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773352.488734:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773352.552726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773352.616726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773352.680720:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773352.680727:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773352.680756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773352.680780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773352.744787:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773352.744875:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773352.744883:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773352.872753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773352.872759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773352.936735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773353.000792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773353.064781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773353.064806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773353.064807:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773353.128794:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773353.192764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773353.192789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773353.192808:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773353.320826:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773353.320826:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773353.320829:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773353.320831:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773353.384776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773353.384816:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773353.384819:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773353.384821:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773353.384872:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773353.384891:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773353.512756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773353.576745:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773353.640724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773353.704785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773353.704788:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773353.704815:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773353.704818:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773353.768787:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773353.768829:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773353.768889:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773353.896765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773353.896790:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773353.960771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773354.024775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773354.088782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773354.088789:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773354.088798:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773354.152731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773354.216766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773354.216791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773354.216801:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773354.344762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773354.344776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773354.344817:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773354.344840:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773354.408765:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773354.408771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773354.408821:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773354.408845:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773354.408871:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773354.408877:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773354.536749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773354.600776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773354.664761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773354.728762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773354.728771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773354.728772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773354.728772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773354.792818:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773354.792852:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773354.792884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773354.920761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773354.920761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773354.984789:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773355.048740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773355.112733:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773355.112733:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773355.112740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773355.176727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773355.240718:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773355.240754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773355.240761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773355.368739:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773355.368739:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773355.368739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773355.368739:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773355.432732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773355.432747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773355.432748:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773355.432753:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773355.432777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773355.432796:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773355.560726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773355.624706:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773355.688703:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773355.752716:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773355.752724:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773355.752743:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773355.752768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773355.816763:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773355.816787:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773355.816837:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773355.944710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773355.944713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773356.008713:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773356.072721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773356.136795:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773356.136795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773356.136796:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773356.200828:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773356.264752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773356.264763:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773356.264809:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773356.392802:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773356.392809:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773356.392811:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773356.392820:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773356.456716:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773356.456723:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773356.456729:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773356.456812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773356.456830:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773356.456856:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773356.584788:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773356.648716:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773356.712748:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773356.712748:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773356.712756:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773356.712761:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773356.712784:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2352 00000100:00080000:3.0:1790773356.712796:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773356.712810:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773356.712869:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534657792/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773356.712907:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773356.712929:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773356.712932:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773356.712938:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2352 00000100:00080000:3.0:1790773356.712943:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773356.712950:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773356.712962:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773356.712964:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534657792:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773356.712979:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534657920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773356.712992:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773356.712996:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773356.712998:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773356.713003:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2352 00000100:00080000:3.0:1790773356.713006:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773356.713012:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773356.713018:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773356.713029:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534658048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773356.713031:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773356.713040:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773356.713044:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773356.713047:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773356.713052:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2352 00000100:00080000:3.0:1790773356.713056:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773356.713062:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773356.713068:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773356.713080:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534658176/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773356.713092:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773356.713094:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773356/real 1790773356] req@ffff9c4ef73c0000 x1877759534657792/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773356.713096:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773356.713107:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773356.713130:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2352 00000100:00100000:1.0:1790773356.713134:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534657792/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773356.713134:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773356.713141:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773356.713147:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773356.713147:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:1.0:1790773356.713152:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773356.713159:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534657792:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773356.713163:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534658304/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773356.713174:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773356.713179:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773356.713181:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773356.713187:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534657920:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773356.713187:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2352 00000100:00080000:3.0:1790773356.713191:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773356.713197:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773356.713202:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773356.713214:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773356.713214:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534658432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773356.713226:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773356.713230:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773356/real 1790773356] req@ffff9c4ef73c1500 x1877759534657920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773356.713231:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773356.713233:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773356.713239:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2352 00000100:00080000:3.0:1790773356.713243:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773356.713262:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534657920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773356.713267:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773356.713275:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773356.713289:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534658560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773356.713290:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773356.713297:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773356.713304:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534657920:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773356.713306:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773356.713312:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1790773356.713315:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534658048:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773356.713315:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773356.713322:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2352 00000100:00080000:3.0:1790773356.713328:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773356.713336:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773356.713360:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790773356.713360:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773356.713371:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534658688/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773356.713373:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773356/real 1790773356] req@ffff9c4ef73c0700 x1877759534658048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773356.713385:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773356.713388:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534658048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773356.713389:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773356.713392:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773356.713397:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2352 00000100:00080000:1.0:1790773356.713400:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773356.713401:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773356.713404:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773356.713407:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773356.713409:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534658048:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790773356.713413:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773356.713417:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534658176:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773356.713423:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534658816/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773356.713437:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773356.713451:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773356/real 1790773356] req@ffff9c4ef73c2a00 x1877759534658176/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773356.713467:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534658176/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773356.713481:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773356.713485:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773356.713490:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534658176:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773356.713497:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534658304:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773356.713516:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773356.713525:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773356/real 1790773356] req@ffff9c4ef73c2d80 x1877759534658304/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773356.713535:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534658304/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773356.713544:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773356.713546:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773356.713550:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534658304:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773356.713567:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534658432:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773356.713581:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773356.713588:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773356/real 1790773356] req@ffff9c4ef73c3800 x1877759534658432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773356.713597:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534658432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773356.713640:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773356.713646:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773356.713653:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534658432:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773356.713663:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534658560:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773356.713728:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773356.713752:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773356/real 1790773356] req@ffff9c4ef73c0a80 x1877759534658560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773356.713772:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534658560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773356.713785:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773356.713807:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773356.713810:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534658560:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773356.713818:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534658688:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773356.713842:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773356.713874:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773356/real 1790773356] req@ffff9c4ef73c0e00 x1877759534658688/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773356.713894:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534658688/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773356.713912:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773356.713918:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773356.713926:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534658688:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773356.713937:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534658816:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773356.713969:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773356.713989:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773356/real 1790773356] req@ffff9c4ef73c2300 x1877759534658816/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773356.714010:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534658816/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773356.714028:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773356.714035:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773356.714041:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534658816:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773356.776766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773356.776766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773356.776768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773356.776861:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773356.840770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773356.840838:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773356.840918:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773356.968779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773356.968808:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773357.032808:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773357.096895:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773357.160755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773357.160755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773357.160804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773357.224790:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773357.288729:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773357.288752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773357.288794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773357.416745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773357.416763:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773357.416786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773357.416790:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773357.480776:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773357.480793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773357.480830:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773357.480853:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773357.480895:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773357.480947:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773357.608743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773357.672706:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773357.736715:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773357.800762:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773357.800762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773357.800768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773357.800774:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773357.864842:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773357.864873:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773357.865007:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773357.992741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773357.992754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773358.056779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773358.120764:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773358.184708:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773358.184733:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773358.184734:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773358.248744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773358.312776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773358.312776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773358.312777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773358.440729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773358.440749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773358.440750:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773358.440753:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773358.504747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773358.504763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773358.504765:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773358.504768:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773358.504800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773358.504811:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773358.632719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773358.696757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773358.760732:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773358.824753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773358.824759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773358.824759:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773358.824759:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773358.888808:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773358.888808:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773358.888908:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773359.016721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773359.016780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773359.080778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773359.144783:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773359.208714:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773359.208715:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773359.208735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773359.272816:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773359.336741:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773359.336794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773359.336794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773359.464751:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773359.464751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773359.464751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773359.464751:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773359.528762:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773359.528764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773359.528773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773359.528792:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773359.528818:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773359.528821:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773359.656831:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773359.720708:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773359.784734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773359.848724:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773359.848740:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773359.848763:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773359.848764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773359.912746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773359.912750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773359.912872:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773360.040763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773360.040791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773360.104771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773360.168838:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773360.232779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773360.232815:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773360.232859:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773360.296780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773360.360789:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773360.360798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773360.360826:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773360.488822:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773360.488822:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773360.488830:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773360.488893:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773360.552855:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773360.552859:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773360.552885:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773360.552911:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773360.552965:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773360.552965:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773360.680776:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773360.744780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773360.808769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773360.872829:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773360.872835:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773360.872849:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773360.872886:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773360.936764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773360.936864:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773360.936911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773361.064766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773361.064778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773361.128818:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773361.192770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773361.256740:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773361.256740:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773361.256822:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773361.320736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773361.384755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773361.384758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773361.384804:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773361.512756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773361.512757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773361.512757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773361.512757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773361.576738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773361.576748:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773361.576748:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773361.576780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773361.576793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773361.576825:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773361.704743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773361.768793:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773361.832755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773361.832784:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773361.832794:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773361.832798:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773361.832808:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2357 00000100:00080000:3.0:1790773361.832818:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773361.832829:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773361.832866:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534658944/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773361.832895:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773361.832901:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773361.832903:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773361.832906:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2357 00000100:00080000:3.0:1790773361.832910:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773361.832916:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773361.832927:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773361.832940:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534659072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773361.832953:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534658944:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773361.832955:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773361.832970:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773361.832972:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773361.832977:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2357 00000100:00080000:3.0:1790773361.832980:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773361.832985:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773361.832990:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773361.833004:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534659200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773361.833041:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773361.833044:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773361.833046:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773361.833050:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2357 00000100:00080000:3.0:1790773361.833055:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773361.833061:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773361.833066:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773361.833075:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534659328/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773361.833081:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773361.833083:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773361.833085:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773361.833088:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2357 00000100:00080000:3.0:1790773361.833090:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773361.833093:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773361.833098:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773361.833102:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773361.833114:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534659456/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773361.833121:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773361.833125:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773361.833126:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773361.833129:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2357 00000100:00080000:3.0:1790773361.833132:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773361.833135:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773361.833140:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773361.833159:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773361/real 1790773361] req@ffff9c4ef73c1c00 x1877759534658944/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773361.833159:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534659584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773361.833239:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773361.833242:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773361.833245:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773361.833251:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534658944/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773361.833251:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2357 00000100:00080000:3.0:1790773361.833262:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773361.833278:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773361.833284:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773361.833284:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773361.833298:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790773361.833318:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534659712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773361.833324:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534658944:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773361.833346:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773361.833350:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773361.833352:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773361.833356:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2357 00000100:00080000:3.0:1790773361.833361:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773361.833365:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773361.833368:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773361.833377:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534659072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773361.833381:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534659840/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773361.833400:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773361.833414:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773361.833416:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773361.833419:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2357 00000100:00080000:3.0:1790773361.833422:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773361.833427:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773361.833432:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:1.0:1790773361.833445:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773361.833446:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534659968/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773361.833493:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773361/real 1790773361] req@ffff9c4ef73c2300 x1877759534659072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773361.833513:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534659072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773361.833534:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773361.833539:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773361.833544:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534659072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773361.833551:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534659200:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773361.833589:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773361.833598:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773361/real 1790773361] req@ffff9c4ef73c0e00 x1877759534659200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773361.833607:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534659200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773361.833620:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773361.833623:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773361.833626:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534659200:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773361.833630:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534659328:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773361.833645:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773361.833652:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773361/real 1790773361] req@ffff9c4ef73c0a80 x1877759534659328/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773361.833661:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534659328/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773361.833669:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773361.833672:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773361.833674:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534659328:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773361.833680:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534659456:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773361.833692:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773361.833699:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773361/real 1790773361] req@ffff9c4ef73c3800 x1877759534659456/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773361.833733:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534659456/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773361.833742:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773361.833745:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773361.833748:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534659456:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773361.833753:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534659584:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773361.833769:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773361.833779:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773361/real 1790773361] req@ffff9c4ef73c2d80 x1877759534659584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773361.833788:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534659584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773361.833801:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773361.833804:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773361.833807:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534659584:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773361.833812:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534659712:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773361.833825:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773361.833832:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773361/real 1790773361] req@ffff9c4ef73c2a00 x1877759534659712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773361.833863:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534659712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773361.833875:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773361.833880:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773361.833885:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534659712:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773361.833930:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534659840:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773361.833954:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773361.833965:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773361/real 1790773361] req@ffff9c4ef73c0700 x1877759534659840/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773361.833975:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534659840/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773361.833983:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773361.833986:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773361.833989:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534659840:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773361.833994:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534659968:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773361.834034:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773361.834051:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773361/real 1790773361] req@ffff9c4ef73c1500 x1877759534659968/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773361.834067:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534659968/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773361.834078:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773361.834081:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773361.834084:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534659968:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773361.896810:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773361.896810:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773361.896817:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773361.896829:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773361.960803:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773361.960805:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773361.960821:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773362.088760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773362.088771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773362.152877:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773362.216793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773362.280739:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773362.280741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773362.280746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773362.344794:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773362.408790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773362.408817:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773362.408818:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773362.536776:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773362.536790:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773362.536807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773362.536807:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773362.600718:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773362.600755:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773362.600758:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773362.600759:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773362.600783:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773362.600804:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773362.728780:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773362.792744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773362.856717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773362.920767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773362.920767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773362.920790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773362.920801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773362.984715:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773362.984733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773362.984870:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773363.112728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773363.112755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773363.176800:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773363.240740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773363.304706:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773363.304714:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773363.304738:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773363.368725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773363.432747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773363.432747:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773363.432747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773363.560719:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773363.560719:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773363.560735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773363.560736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773363.624721:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773363.624727:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773363.624730:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773363.624750:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773363.624766:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773363.624781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773363.752728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773363.816750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773363.880741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773363.944750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773363.944751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773363.944751:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773363.944760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773364.008717:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773364.008724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773364.008837:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773364.136737:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773364.136776:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773364.200758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773364.264836:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773364.328724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773364.328725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773364.328728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773364.392771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773364.456711:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773364.456730:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773364.456732:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773364.584741:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773364.584753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773364.584758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773364.584780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773364.648718:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773364.648722:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773364.648745:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773364.648745:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773364.648766:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773364.648777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773364.776740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773364.840725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773364.904742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773364.968734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773364.968740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773364.968742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773364.968742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773365.032731:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773365.032762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773365.032793:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773365.160724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773365.160724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773365.224780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773365.288712:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773365.352726:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773365.352741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773365.352752:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773365.416913:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773365.480802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773365.480814:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773365.480815:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773365.608780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773365.608780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773365.608790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773365.608857:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773365.672727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773365.672731:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773365.672817:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773365.672857:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773365.672865:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773365.672901:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773365.801704:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773365.864798:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773365.928725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773365.992820:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773365.992820:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773365.992843:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773365.992846:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773366.056738:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773366.056880:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773366.056888:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773366.184746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773366.184761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773366.248825:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773366.312797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773366.376782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773366.376786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773366.376793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773366.440754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773366.504765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773366.504777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773366.504809:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773366.632813:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773366.632813:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773366.632814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773366.632819:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773366.696739:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773366.696740:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773366.696758:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773366.696768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773366.696790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773366.696802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773366.824733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773366.888751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773366.952736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773366.952743:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773366.952749:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773366.952753:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773366.952762:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2362 00000100:00080000:2.0:1790773366.952778:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773366.952788:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773366.952822:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534660096/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773366.952860:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773366.952864:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773366.952865:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773366.952869:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2362 00000100:00080000:2.0:1790773366.952873:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773366.952877:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773366.952885:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773366.952890:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534660096:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773366.952898:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534660224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773366.952908:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773366.952923:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773366.952925:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773366.952929:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2362 00000100:00080000:2.0:1790773366.952932:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773366.952946:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773366.952950:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773366.952962:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534660352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773366.952987:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773366.952990:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773366.952992:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773366.952996:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2362 00000100:00080000:2.0:1790773366.953013:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773366.953019:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773366.953022:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773366.953030:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773366.953043:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534660480/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773366.953052:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773366.953056:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773366.953058:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773366.953062:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2362 00000100:00080000:2.0:1790773366.953067:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773366.953073:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773366/real 1790773366] req@ffff9c4efbcc0380 x1877759534660096/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773366.953077:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773366.953086:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773366.953102:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534660608/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773366.953106:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534660096/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773366.953121:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773366.953127:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773366.953129:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773366.953133:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1790773366.953155:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534660096:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773366.953160:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773366.953167:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2362 00000100:00080000:2.0:1790773366.953170:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773366.953175:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773366.953189:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773366.953200:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534660224:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773366.953200:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534660736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773366.953215:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773366.953218:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773366.953220:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773366.953223:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2362 00000100:00080000:2.0:1790773366.953245:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773366.953251:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773366.953257:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773366.953259:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773366.953273:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534660864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773366.953283:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773366.953285:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773366/real 1790773366] req@ffff9c4efbcc3100 x1877759534660224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773366.953288:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773366.953292:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773366.953303:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2362 00000100:00080000:2.0:1790773366.953310:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773366.953316:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773366.953321:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773366.953325:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534660224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773366.953334:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534660992/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773366.953358:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773366.953362:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773366.953364:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773366.953368:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773366.953371:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2362 00000100:00080000:1.0:1790773366.953377:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773366.953381:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773366.953385:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534660224:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773366.953394:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773366.953397:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534660352:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790773366.953405:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773366.953429:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534661120/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773366.953431:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773366.953462:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773366/real 1790773366] req@ffff9c4efbcc0700 x1877759534660352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773366.953473:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534660352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773366.953480:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773366.953483:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773366.953486:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534660352:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773366.953490:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534660480:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773366.953502:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773366.953509:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773366/real 1790773366] req@ffff9c4efbcc2680 x1877759534660480/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773366.953516:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534660480/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773366.953523:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773366.953525:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773366.953527:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534660480:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773366.953531:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534660608:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773366.953559:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773366.953571:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773366/real 1790773366] req@ffff9c4efbcc0a80 x1877759534660608/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773366.953596:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534660608/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773366.953607:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773366.953627:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773366.953629:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534660608:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773366.953646:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534660736:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773366.953657:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773366.953663:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773366/real 1790773366] req@ffff9c4efbcc2300 x1877759534660736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773366.953671:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534660736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773366.953682:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773366.953684:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773366.953686:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534660736:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773366.953690:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534660864:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773366.953699:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773366.953705:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773366/real 1790773366] req@ffff9c4efbcc1880 x1877759534660864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773366.953712:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534660864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773366.953723:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773366.953725:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773366.953727:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534660864:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773366.953748:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534660992:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773366.953760:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773366.953767:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773366/real 1790773366] req@ffff9c4efbcc1500 x1877759534660992/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773366.953775:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534660992/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773366.953783:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773366.953786:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773366.953788:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534660992:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773366.953792:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534661120:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773366.953802:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773366.953831:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773366/real 1790773366] req@ffff9c4efbcc1f80 x1877759534661120/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773366.953845:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534661120/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773366.953858:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773366.953862:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773366.953867:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534661120:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773367.016746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773367.016746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773367.016747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773367.016747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773367.080774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773367.080780:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773367.080857:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773367.208723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773367.208752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773367.272755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773367.336734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773367.400752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773367.400753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773367.400754:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773367.464717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773367.528779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773367.528779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773367.528781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773367.656778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773367.656808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773367.656829:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773367.656862:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773367.720713:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773367.720739:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773367.720739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773367.720759:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773367.720770:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773367.720771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773367.848717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773367.912733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773367.976709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773368.040734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773368.040735:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773368.040735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773368.040735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773368.104752:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773368.104752:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773368.104829:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773368.232796:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773368.232798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773368.296755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773368.360800:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773368.424809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773368.424831:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773368.424836:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773368.488746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773368.552795:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773368.552797:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773368.552797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773368.680769:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773368.680774:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773368.680778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773368.680778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773368.744757:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773368.744765:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773368.744774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773368.744799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773368.744817:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773368.744833:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773368.872721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773368.936738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773369.000725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773369.064750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773369.064750:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773369.064818:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773369.064818:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773369.128810:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773369.128813:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773369.128824:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773369.256748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773369.256782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773369.320753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773369.384748:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773369.448779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773369.448794:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773369.448795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773369.512733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773369.576742:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773369.576742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773369.576798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773369.704750:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773369.704750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773369.704792:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773369.704793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773369.768748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773369.768749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773369.768756:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773369.768792:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773369.768800:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773369.768808:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773369.896749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773369.960726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773370.024751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773370.088750:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773370.088805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773370.088805:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773370.088809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773370.152763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773370.152806:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773370.152990:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773370.280740:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773370.280773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773370.344722:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773370.408756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773370.472759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773370.472762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773370.472762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773370.536762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773370.600766:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773370.600767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773370.600767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773370.728829:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773370.728830:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773370.728830:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773370.728913:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773370.792728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773370.792745:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773370.792751:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773370.792778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773370.792783:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773370.792789:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773370.920868:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773370.984756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773371.048721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773371.112732:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773371.112736:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773371.112738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773371.112739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773371.176727:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773371.176728:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773371.176806:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773371.304793:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773371.304793:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773371.368779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773371.432812:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773371.496731:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773371.496735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773371.496756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773371.560772:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773371.624804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773371.624810:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773371.624856:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773371.752773:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773371.752774:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773371.752791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773371.752869:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773371.816776:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773371.816838:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773371.816846:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773371.816851:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773371.816870:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773371.816888:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773371.944787:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773372.008765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773372.072790:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773372.072806:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773372.072844:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773372.072847:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773372.072856:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2367 00000100:00080000:2.0:1790773372.072865:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773372.072874:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773372.072913:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534661248/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773372.072940:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773372.072943:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773372.072945:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773372.072949:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2367 00000100:00080000:2.0:1790773372.072953:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773372.072958:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773372.072967:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773372.072973:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534661248:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773372.072980:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534661376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773372.072989:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773372.072991:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773372.072993:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773372.072997:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2367 00000100:00080000:2.0:1790773372.073000:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773372.073003:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773372.073007:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773372.073016:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534661504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773372.073022:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773372.073025:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773372.073026:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773372.073029:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2367 00000100:00080000:2.0:1790773372.073032:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773372.073035:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773372.073038:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:1.0:1790773372.073044:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773372.073048:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534661632/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773372.073056:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773372.073059:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773372.073061:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773372.073064:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2367 00000100:00080000:2.0:1790773372.073066:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773372.073069:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773372.073072:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773372.073082:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534661760/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773372.073086:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773372/real 1790773372] req@ffff9c4efbcc1c00 x1877759534661248/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773372.073099:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773372.073103:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773372.073106:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773372.073112:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534661248/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773372.073113:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2367 00000100:00080000:2.0:1790773372.073118:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773372.073124:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773372.073130:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790773372.073130:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773372.073137:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773372.073140:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534661888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773372.073144:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534661248:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773372.073156:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773372.073159:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773372.073161:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773372.073165:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2367 00000100:00080000:2.0:1790773372.073168:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773372.073175:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534661376:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773372.073175:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773372.073183:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773372.073193:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534662016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773372.073201:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773372.073202:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773372.073207:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773372.073210:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773372.073214:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2367 00000100:00100000:1.0:1790773372.073218:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773372/real 1790773372] req@ffff9c4efbcc2a00 x1877759534661376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773372.073219:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773372.073227:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773372.073235:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773372.073240:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534661376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773372.073247:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534662144/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773372.073275:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773372.073278:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773372.073279:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773372.073283:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773372.073283:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2367 00000100:00080000:2.0:1790773372.073289:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773372.073292:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773372.073296:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773372.073300:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534661376:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790773372.073302:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773372.073312:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534661504:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773372.073317:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534662272/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773372.073333:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773372.073343:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773372/real 1790773372] req@ffff9c4efbcc1f80 x1877759534661504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773372.073352:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534661504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773372.073359:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773372.073361:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773372.073364:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534661504:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773372.073367:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534661632:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773372.073378:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773372.073383:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773372/real 1790773372] req@ffff9c4efbcc1500 x1877759534661632/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773372.073391:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534661632/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773372.073398:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773372.073400:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773372.073403:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534661632:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773372.073407:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534661760:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773372.073417:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773372.073423:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773372/real 1790773372] req@ffff9c4efbcc1880 x1877759534661760/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773372.073431:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534661760/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773372.073438:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773372.073440:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773372.073442:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534661760:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773372.073446:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534661888:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773372.073455:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773372.073462:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773372/real 1790773372] req@ffff9c4efbcc2300 x1877759534661888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773372.073470:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534661888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773372.073481:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773372.073483:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773372.073485:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534661888:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773372.073499:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534662016:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773372.073510:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773372.073517:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773372/real 1790773372] req@ffff9c4efbcc0a80 x1877759534662016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773372.073525:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534662016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773372.073531:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773372.073533:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773372.073535:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534662016:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773372.073538:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534662144:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773372.073548:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773372.073554:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773372/real 1790773372] req@ffff9c4efbcc2680 x1877759534662144/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773372.073562:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534662144/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773372.073570:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773372.073572:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773372.073574:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534662144:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773372.073577:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534662272:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773372.073587:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773372.073592:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773372/real 1790773372] req@ffff9c4efbcc0700 x1877759534662272/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773372.073600:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534662272/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773372.073608:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773372.073629:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773372.073634:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534662272:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773372.136723:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773372.136727:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773372.136735:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773372.136756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773372.200728:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773372.200730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773372.200761:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773372.328759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773372.328768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773372.392711:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773372.456714:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773372.520737:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773372.520737:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773372.520737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773372.584755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773372.648785:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773372.648854:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773372.648868:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773372.776880:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773372.776882:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773372.776883:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773372.776884:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773372.840748:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773372.840749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773372.840756:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773372.840793:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773372.840816:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773372.840819:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773372.968714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773373.032712:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773373.096713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773373.160737:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773373.160757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773373.160781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773373.160805:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773373.224857:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773373.224874:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773373.225029:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773373.352745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773373.352761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773373.416720:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773373.480756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773373.544722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773373.544759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773373.544806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773373.608727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773373.672728:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773373.672728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773373.672797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773373.800810:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773373.800829:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773373.800836:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773373.800880:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773373.864814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773373.864818:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773373.864856:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773373.864873:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773373.864884:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773373.864915:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773373.992761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773374.056832:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773374.120757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773374.184767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773374.184767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773374.184768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773374.184787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773374.248726:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773374.248731:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773374.248907:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773374.376723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773374.376723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773374.440732:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773374.504735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773374.568809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773374.568810:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773374.568815:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773374.632724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773374.696736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773374.696736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773374.696739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773374.824731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773374.824740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773374.824749:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773374.824749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773374.888805:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773374.888815:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773374.888871:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773374.888876:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773374.888910:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773374.888913:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773375.016737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773375.080836:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773375.144743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773375.208819:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773375.208826:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773375.208834:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773375.208876:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773375.272771:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773375.272795:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773375.272959:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773375.400781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773375.400781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773375.464739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773375.528788:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773375.592727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773375.592791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773375.592825:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773375.656821:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773375.720760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773375.720761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773375.720793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773375.848763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773375.848777:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773375.848800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773375.848840:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773375.912734:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773375.912759:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773375.912761:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773375.912782:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773375.912813:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773375.912815:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773376.040752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773376.104748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773376.168754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773376.232757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773376.232757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773376.232757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773376.232758:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773376.296760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773376.296760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773376.296788:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773376.424755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773376.424770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773376.488782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773376.552722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773376.616741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773376.616742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773376.616753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773376.680757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773376.744790:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773376.744808:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773376.744812:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773376.872756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773376.872756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773376.872756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773376.872758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773376.936700:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773376.936733:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773376.936762:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773376.936768:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773376.936770:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773376.936771:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773377.064759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773377.128757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773377.192729:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773377.192748:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773377.192753:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773377.192762:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2373 00000100:00080000:2.0:1790773377.192772:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773377.192783:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773377.192802:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773377.192821:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534662400/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773377.192841:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773377.192846:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773377.192849:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773377.192853:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2373 00000100:00080000:2.0:1790773377.192856:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773377.192859:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773377.192868:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773377.192878:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534662528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773377.192888:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534662400:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773377.192894:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773377.192902:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773377.192904:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773377.192908:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2373 00000100:00080000:2.0:1790773377.192910:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773377.192913:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773377.192920:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773377.192933:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534662656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773377.192939:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773377.192943:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773377.192945:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773377.192948:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2373 00000100:00080000:2.0:1790773377.192951:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773377.192954:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773377.192960:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773377.192968:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534662784/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773377.192976:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773377.192979:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773377.192980:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773377.192983:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2373 00000100:00080000:2.0:1790773377.192986:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773377.192997:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773377.193001:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790773377.193006:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773377.193017:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534662912/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773377.193026:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773377.193029:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773377.193031:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773377.193035:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2373 00000100:00080000:2.0:1790773377.193038:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773377.193043:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773377.193047:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773377.193056:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534663040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773377.193063:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773377.193066:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773377.193074:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773377.193076:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773377/real 1790773377] req@ffff9c4efbcc3100 x1877759534662400/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773377.193081:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2373 00000100:00080000:2.0:1790773377.193086:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773377.193099:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773377.193110:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773377.193126:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534663168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773377.193135:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773377.193139:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773377.193142:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773377.193149:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534662400/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773377.193149:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2373 00000100:00080000:2.0:1790773377.193162:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773377.193177:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773377.193184:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773377.193189:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1790773377.193194:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534663296/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773377.193201:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773377.193212:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773377.193217:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773377.193219:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773377.193235:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534662400:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773377.193236:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2373 00000100:00080000:2.0:1790773377.193256:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773377.193268:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773377.193275:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773377.193286:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534662528:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773377.193295:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534663424/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773377.193323:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773377.193343:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773377/real 1790773377] req@ffff9c4efbcc0380 x1877759534662528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773377.193358:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534662528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773377.193385:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773377.193390:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773377.193393:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534662528:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773377.193401:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534662656:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773377.193418:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773377.193429:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773377/real 1790773377] req@ffff9c4efbcc0700 x1877759534662656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773377.193440:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534662656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773377.193449:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773377.193453:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773377.193456:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534662656:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773377.193461:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534662784:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773377.193477:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773377.193487:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773377/real 1790773377] req@ffff9c4efbcc2680 x1877759534662784/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773377.193498:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534662784/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773377.193508:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773377.193511:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773377.193515:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534662784:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773377.193522:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534662912:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773377.193537:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773377.193546:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773377/real 1790773377] req@ffff9c4efbcc0a80 x1877759534662912/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773377.193557:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534662912/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773377.193566:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773377.193569:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773377.193573:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534662912:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773377.193578:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534663040:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773377.193594:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773377.193603:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773377/real 1790773377] req@ffff9c4efbcc2300 x1877759534663040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773377.193615:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534663040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773377.193643:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773377.193646:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773377.193677:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534663040:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773377.193685:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534663168:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773377.193704:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773377.193746:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773377/real 1790773377] req@ffff9c4efbcc1880 x1877759534663168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773377.193770:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534663168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773377.193779:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773377.193783:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773377.193787:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534663168:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773377.193793:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534663296:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773377.193815:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773377.193825:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773377/real 1790773377] req@ffff9c4efbcc1500 x1877759534663296/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773377.193837:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534663296/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773377.193847:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773377.193850:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773377.193854:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534663296:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773377.193881:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534663424:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773377.193898:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773377.193907:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773377/real 1790773377] req@ffff9c4efbcc1f80 x1877759534663424/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773377.193919:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534663424/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773377.193930:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773377.193933:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773377.193937:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534663424:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773377.256751:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773377.256751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773377.256771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773377.256823:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773377.320734:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773377.320870:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773377.320925:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773377.448765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773377.448773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773377.512866:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773377.576779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773377.640786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773377.640787:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773377.640814:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773377.704760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773377.768818:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773377.768860:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773377.768875:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773377.896813:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773377.896814:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773377.896853:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773377.896863:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773377.960784:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773377.960802:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773377.960826:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773377.960845:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773377.960862:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773377.960862:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773378.088792:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773378.152768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773378.216825:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773378.280827:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773378.280854:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773378.280867:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773378.280871:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773378.344819:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773378.344837:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773378.344962:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773378.472811:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773378.472840:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773378.536851:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773378.600934:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773378.664790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773378.664811:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773378.664850:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773378.728795:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773378.792802:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773378.792847:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773378.792875:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773378.920817:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773378.920862:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773378.920869:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773378.920870:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773378.984843:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773378.984843:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773378.984849:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773378.984893:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773378.984936:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773378.984958:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773379.112773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773379.176746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773379.240737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773379.304784:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773379.304785:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773379.304786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773379.304786:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773379.368781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773379.368787:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773379.368890:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773379.496743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773379.496744:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773379.560795:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773379.624801:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773379.688730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773379.688780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773379.688780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773379.752740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773379.816734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773379.816827:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773379.816863:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773379.944820:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773379.944831:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773379.944836:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773379.944873:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773380.008750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773380.008795:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773380.008826:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773380.008829:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773380.008829:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773380.008830:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773380.136758:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773380.200744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773380.264731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773380.328770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773380.328801:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773380.328806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773380.328808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773380.392758:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773380.392791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773380.392820:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773380.520738:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773380.520741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773380.584739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773380.648776:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773380.712774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773380.712774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773380.712834:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773380.776753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773380.840720:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773380.840749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773380.840802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773380.968802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773380.968802:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773380.968816:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773380.968872:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773381.032735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773381.032766:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773381.032779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773381.032779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773381.032791:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773381.032818:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773381.160761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773381.224760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773381.288757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773381.352766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773381.352766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773381.352810:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773381.352811:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773381.416791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773381.416797:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773381.416881:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773381.544804:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773381.544810:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773381.608719:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773381.672787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773381.736765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773381.736765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773381.736783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773381.800724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773381.864748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773381.864777:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773381.864779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773381.992737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773381.992753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773381.992781:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773381.992785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773382.056729:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773382.056740:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773382.056796:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773382.056796:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773382.056798:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773382.056798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773382.184746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773382.248756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773382.312739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773382.312799:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773382.312822:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773382.312827:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773382.312837:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2378 00000100:00080000:0.0:1790773382.312847:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773382.312858:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773382.312895:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534663552/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773382.312923:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773382.312928:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773382.312932:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773382.312938:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2378 00000100:00080000:0.0:1790773382.312943:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773382.312950:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773382.312962:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773382.312964:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534663552:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773382.312980:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534663680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773382.312993:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773382.312998:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773382.313000:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773382.313005:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2378 00000100:00080000:0.0:1790773382.313020:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773382.313026:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773382.313031:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773382.313034:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773382.313047:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534663808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773382.313060:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773382.313064:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773382.313067:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773382.313072:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2378 00000100:00080000:0.0:1790773382.313076:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773382.313083:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773382.313089:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773382.313090:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773382/real 1790773382] req@ffff9c4ec7a7f480 x1877759534663552/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773382.313118:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534663936/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773382.313128:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534663552/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773382.313130:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773382.313136:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773382.313139:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773382.313143:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773382.313144:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2378 00000100:00080000:0.0:1790773382.313149:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773382.313149:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773382.313156:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773382.313156:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534663552:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773382.313163:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773382.313174:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534664064/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773382.313183:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534663680:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773382.313188:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773382.313193:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773382.313197:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773382.313215:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2378 00000800:00000100:1.0:1790773382.313215:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773382.313221:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773382.313229:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773382.313233:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773382/real 1790773382] req@ffff9c4ec7a7df80 x1877759534663680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773382.313236:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773382.313251:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534664192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773382.313263:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534663680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773382.313265:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773382.313270:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773382.313273:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773382.313279:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2378 00000100:00080000:0.0:1790773382.313283:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773382.313287:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773382.313289:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773382.313292:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790773382.313295:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773382.313298:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534663680:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773382.313307:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534663808:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773382.313308:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534664320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773382.313321:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773382.313325:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773382.313328:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773382.313333:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2378 00000800:00000100:1.0:1790773382.313333:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773382.313338:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773382.313345:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773382.313349:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773382/real 1790773382] req@ffff9c4ec7a7ed80 x1877759534663808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773382.313352:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773382.313368:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534664448/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773382.313368:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534663808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773382.313383:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773382.313383:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773382.313388:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773382.313393:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534663808:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773382.313401:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773382.313402:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534663936:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773382.313404:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773382.313411:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2378 00000100:00080000:0.0:1790773382.313415:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773382.313429:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773382.313432:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773382.313439:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773382.313444:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773382/real 1790773382] req@ffff9c4ec7a7d880 x1877759534663936/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773382.313453:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534664576/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773382.313462:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534663936/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773382.313486:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773382.313491:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773382.313497:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534663936:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773382.313520:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534664064:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773382.313547:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773382.313559:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773382/real 1790773382] req@ffff9c4ec7a7c380 x1877759534664064/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773382.313570:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534664064/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773382.313580:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773382.313583:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773382.313587:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534664064:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773382.313593:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534664192:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773382.313608:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773382.313656:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773382/real 1790773382] req@ffff9c4ec7a7ca80 x1877759534664192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773382.313689:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534664192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773382.313719:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773382.313722:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773382.313727:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534664192:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773382.313733:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534664320:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773382.313754:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773382.313765:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773382/real 1790773382] req@ffff9c4ec7a7fb80 x1877759534664320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773382.313776:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534664320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773382.313784:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773382.313787:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773382.313791:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534664320:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773382.313796:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534664448:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773382.313826:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773382.313834:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773382/real 1790773382] req@ffff9c4ec7a7e300 x1877759534664448/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773382.313843:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534664448/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773382.313853:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773382.313855:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773382.313858:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534664448:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773382.313863:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534664576:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773382.313876:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773382.313884:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773382/real 1790773382] req@ffff9c4ec7a7c700 x1877759534664576/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773382.313894:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534664576/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773382.313902:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773382.313904:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773382.313907:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534664576:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773382.376818:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773382.376826:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773382.376844:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773382.376846:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773382.440844:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773382.440918:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773382.440944:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773382.568748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773382.568756:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773382.632761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773382.696805:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773382.760775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773382.760790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773382.760796:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773382.824796:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773382.888777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773382.888777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773382.888802:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773383.016781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773383.016781:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773383.016785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773383.016786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773383.080752:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773383.080759:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773383.080763:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773383.080809:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773383.080815:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773383.080826:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773383.208774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773383.272716:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773383.336827:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773383.400830:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773383.400836:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773383.400849:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773383.400852:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773383.464816:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773383.464818:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773383.464819:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773383.592807:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773383.592815:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773383.656714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773383.720828:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773383.784780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773383.784780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773383.784791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773383.848749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773383.912788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773383.912788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773383.912805:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773384.040782:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773384.040795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773384.040848:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773384.040849:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773384.104736:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773384.104738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773384.104752:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773384.104788:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773384.104803:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773384.104821:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773384.232718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773384.296760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773384.360714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773384.424741:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773384.424813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773384.424814:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773384.424823:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773384.488801:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773384.488801:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773384.488835:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773384.616813:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773384.616834:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773384.680781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773384.744773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773384.808737:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773384.808754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773384.808776:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773384.872749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773384.936784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773384.936784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773384.936785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773385.064735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773385.064736:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773385.064740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773385.064741:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773385.128760:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773385.128761:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773385.128763:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773385.128767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773385.128795:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773385.128801:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773385.256764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773385.320743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773385.384744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773385.448736:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773385.448739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773385.448743:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773385.448744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773385.512723:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773385.512762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773385.512795:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773385.640730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773385.640823:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773385.704756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773385.768858:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773385.832849:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773385.832865:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773385.832866:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773385.896735:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773385.960722:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773385.960730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773385.960740:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773386.088784:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773386.088798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773386.088819:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773386.088821:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773386.152749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773386.152755:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773386.152777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773386.152779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773386.152800:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773386.152802:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773386.280739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773386.344728:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773386.408749:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773386.472740:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773386.472740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773386.472743:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773386.472750:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773386.536795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773386.536802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773386.536887:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773386.664714:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773386.664723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773386.728818:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773386.792736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773386.856747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773386.856747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773386.856749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773386.920719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773386.984739:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773386.984739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773386.984739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773387.112737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773387.112738:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773387.112743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773387.112769:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773387.176755:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773387.176756:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773387.176757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773387.176765:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773387.176786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773387.176802:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773387.304730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773387.368781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773387.432737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773387.432754:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773387.432763:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773387.432768:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773387.432778:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2383 00000100:00080000:2.0:1790773387.432789:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773387.432801:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773387.432843:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534664704/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773387.432871:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773387.432876:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773387.432878:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773387.432884:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2383 00000100:00080000:2.0:1790773387.432889:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773387.432896:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773387.432900:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534664704:192.168.204.150@tcp:250:kworker.0 00000008:00100000:2.0:1790773387.432908:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773387.432926:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534664832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773387.432939:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773387.432944:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773387.432946:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773387.432951:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2383 00000100:00080000:2.0:1790773387.432956:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773387.432963:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773387.432970:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773387.432987:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534664960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773387.432998:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534664832:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773387.433000:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773387.433009:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773387.433011:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773387.433017:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2383 00000100:00080000:2.0:1790773387.433024:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773387.433032:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773387.433039:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773387.433041:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534664960:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773387.433058:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534665088/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773387.433078:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773387.433083:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773387.433085:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773387.433091:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2383 00000100:00080000:2.0:1790773387.433097:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773387.433100:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534665088:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773387.433106:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773387.433114:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773387.433130:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534665216/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773387.433160:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773387.433165:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773387.433168:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773387.433172:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534665216:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773387.433176:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2383 00000100:00080000:2.0:1790773387.433182:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773387.433190:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773387.433197:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773387.433211:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534665344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773387.433226:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773387.433230:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773387.433233:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773387.433240:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534665344:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773387.433240:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2383 00000100:00080000:2.0:1790773387.433249:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773387.433256:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773387.433263:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773387.433277:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534665472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773387.433298:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773387.433301:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773387.433303:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773387.433308:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2383 00000100:00080000:2.0:1790773387.433313:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773387.433319:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534665472:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773387.433321:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773387.433332:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773387.433347:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534665600/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773387.433359:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773387.433363:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773387.433366:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773387.433371:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2383 00000100:00100000:1.0:1790773387.433376:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534665600:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773387.433377:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773387.433386:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773387.433393:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773387.433407:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534665728/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773387.433447:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534665728:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773387.496722:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773387.496727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773387.496732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773387.496745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773387.560726:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773387.560759:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773387.560784:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773387.688703:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773387.688734:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773387.752789:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773387.816782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773387.880768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773387.880768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773387.880769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773387.944737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773388.008713:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773388.008737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773388.008750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773388.136769:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773388.136771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773388.136772:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773388.136774:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773388.200745:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773388.200750:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773388.200753:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773388.200757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773388.200781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773388.200783:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773388.328753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773388.392765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773388.456733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773388.520757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773388.520757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773388.520757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773388.520758:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773388.584714:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773388.584726:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773388.584785:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773388.712750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773388.712791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773388.776759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773388.840741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773388.904753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773388.904753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773388.904754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773388.968732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773389.032770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773389.032770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773389.032771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773389.160734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773389.160747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773389.160749:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773389.160749:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773389.224724:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773389.224742:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773389.224754:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773389.224790:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773389.224793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773389.224801:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773389.352729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773389.416717:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773389.480773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773389.544747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773389.544747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773389.544759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773389.544759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773389.608727:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773389.608751:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773389.608809:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773389.736779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773389.736787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773389.800785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773389.864710:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773389.928780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773389.928829:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773389.928830:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773389.992738:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773390.056763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773390.056784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773390.056801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773390.184833:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773390.184848:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773390.184885:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773390.184895:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773390.248755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773390.248801:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773390.248802:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773390.248823:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773390.248829:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773390.248862:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773390.376724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773390.440929:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773390.504752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:3.0:1790773390.504951:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:3.0:1790773390.504998:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1790773390.505019:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773390.505078:0:2385:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:3.0:1790773390.505091:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773390.505101:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773390.505109:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773390.505117:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:1.0:1790773390.505118:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773387/real 1790773390] req@ffff9c4efbcc0700 x1877759534665728/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1790773390.505126:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773390.505137:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773390.505147:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:1.0:1790773390.505166:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534665728/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773390.505184:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773390.505191:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773390.505198:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534665728:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773390.505206:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773387/real 1790773390] req@ffff9c4efbcc2680 x1877759534665600/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773390.505219:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534665600/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773390.505227:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773390.505230:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773390.505233:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534665600:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773390.505240:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773387/real 1790773390] req@ffff9c4efbcc0a80 x1877759534665472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773390.505251:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534665472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773390.505261:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773390.505268:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773390.505271:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534665472:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773390.505284:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773387/real 1790773390] req@ffff9c4efbcc2300 x1877759534665344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773390.505300:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534665344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773390.505335:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773390.505348:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773390.505353:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534665344:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773390.505365:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773387/real 1790773390] req@ffff9c4efbcc1880 x1877759534665216/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773390.505388:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534665216/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773390.505408:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773390.505416:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773390.505420:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534665216:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773390.505432:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773387/real 1790773390] req@ffff9c4efbcc1500 x1877759534665088/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773390.505446:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534665088/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773390.505475:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773390.505480:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773390.505484:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534665088:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773390.505494:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773387/real 1790773390] req@ffff9c4efbcc1f80 x1877759534664960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773390.505514:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534664960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773390.505526:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773390.505532:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773390.505536:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534664960:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773390.505548:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773387/real 1790773390] req@ffff9c4efbcc1c00 x1877759534664832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773390.505562:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534664832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773390.505591:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773390.505598:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773390.505602:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534664832:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773390.505613:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773387/real 1790773390] req@ffff9c4efbcc2a00 x1877759534664704/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773390.505628:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534664704/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773390.505638:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773390.505643:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773390.505647:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534664704:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773390.568764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773390.568764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773390.568837:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773390.568838:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773390.632762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773390.632768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773390.632855:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773390.760731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773390.760750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773390.824773:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773390.888747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773390.952762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773390.952764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773390.952765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773391.016737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773391.080742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773391.080757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773391.080757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773391.208759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773391.208780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773391.208800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773391.208800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773391.272773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773391.272786:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773391.272795:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773391.272810:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773391.272838:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773391.272844:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773391.400747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773391.464751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773391.528760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773391.592746:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773391.592779:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773391.592786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773391.592790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773391.656801:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773391.656845:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773391.656886:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773391.784771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773391.784800:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773391.848774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773391.912797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773391.976753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773391.976783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773391.976805:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773392.040745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773392.104796:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773392.104798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773392.104801:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773392.232771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773392.232772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773392.232793:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773392.232793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773392.296757:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773392.296789:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773392.296799:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773392.296812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773392.296835:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773392.296867:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773392.424756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773392.488830:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773392.552770:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773392.552770:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773392.552785:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773392.552791:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773392.552804:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2388 00000100:00080000:2.0:1790773392.552818:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773392.552835:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773392.552882:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534665856/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773392.552925:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773392.552931:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773392.552934:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773392.552940:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2388 00000100:00080000:2.0:1790773392.552946:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773392.552952:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773392.552965:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773392.552992:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534665984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773392.553006:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534665856:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773392.553027:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773392.553036:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773392.553041:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773392.553081:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2388 00000100:00080000:2.0:1790773392.553092:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773392.553104:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773392.553114:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790773392.553123:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773392.553157:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534666112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773392.553181:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773392.553189:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1790773392.553190:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773392/real 1790773392] req@ffff9c4efbcc0380 x1877759534665856/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773392.553193:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773392.553209:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2388 00000100:00080000:2.0:1790773392.553222:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773392.553236:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773392.553247:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773392.553252:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534665856/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773392.553277:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1790773392.553279:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534666240/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773392.553292:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773392.553310:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534665856:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773392.553345:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773392.553353:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773392.553358:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773392.553366:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534665984:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773392.553374:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2388 00000100:00080000:2.0:1790773392.553388:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773392.553401:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773392.553443:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:1.0:1790773392.553459:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773392.553492:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773392/real 1790773392] req@ffff9c4efbcc3100 x1877759534665984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773392.553499:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534666368/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773392.553556:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773392.553563:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773392.553567:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773392.553576:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2388 00000100:00080000:2.0:1790773392.553584:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773392.553587:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534665984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773392.553602:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773392.553622:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773392.553641:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1790773392.553694:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534666496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773392.553714:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773392.553722:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773392.553726:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773392.553739:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773392.553742:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2388 00000100:00080000:2.0:1790773392.553757:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773392.553759:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534665984:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773392.553774:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773392.553784:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534666112:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790773392.553792:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773392.553827:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534666624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773392.553849:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773392.553851:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773392.553861:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773392.553866:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773392.553878:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2388 00000100:00080000:2.0:1790773392.553890:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773392.553897:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773392/real 1790773392] req@ffff9c4efbcc2a00 x1877759534666112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773392.553905:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773392.553922:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773392.553945:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534666112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773392.553956:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534666752/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773392.553980:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773392.553991:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773392.553996:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773392.554008:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773392.554014:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534666112:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773392.554017:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773392.554037:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2388 00000100:00100000:1.0:1790773392.554045:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534666240:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773392.554062:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773392.554079:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773392.554090:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:1.0:1790773392.554106:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773392.554119:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534666880/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773392.554139:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773392/real 1790773392] req@ffff9c4efbcc1c00 x1877759534666240/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773392.554195:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534666240/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773392.554219:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773392.554228:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773392.554236:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534666240:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773392.554274:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534666368:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773392.554323:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773392.554354:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773392/real 1790773392] req@ffff9c4efbcc1f80 x1877759534666368/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773392.554381:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534666368/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773392.554401:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773392.554409:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773392.554444:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534666368:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773392.554457:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534666496:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773392.554519:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773392.554545:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773392/real 1790773392] req@ffff9c4efbcc1500 x1877759534666496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773392.554569:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534666496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773392.554601:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773392.554608:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773392.554617:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534666496:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773392.554631:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534666624:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773392.554715:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773392.554741:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773392/real 1790773392] req@ffff9c4efbcc1880 x1877759534666624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773392.554768:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534666624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773392.554786:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773392.554793:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773392.554801:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534666624:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773392.554815:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534666752:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773392.554857:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773392.554881:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773392/real 1790773392] req@ffff9c4efbcc2300 x1877759534666752/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773392.554907:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534666752/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773392.554928:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773392.554935:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773392.554943:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534666752:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773392.554956:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534666880:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773392.554995:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773392.555020:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773392/real 1790773392] req@ffff9c4efbcc0a80 x1877759534666880/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773392.555045:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534666880/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773392.555065:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773392.555073:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773392.555082:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534666880:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773392.616749:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773392.616774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773392.616774:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773392.616790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773392.680787:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773392.680830:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773392.680886:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773392.808792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773392.808793:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773392.872743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773392.936869:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773393.000757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773393.000772:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773393.000799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773393.064753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773393.128780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773393.128802:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773393.128836:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773393.256747:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773393.256763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773393.256784:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773393.256787:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773393.320719:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773393.320730:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773393.320735:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773393.320796:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773393.320809:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773393.320816:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773393.448703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773393.512737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773393.576717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773393.640740:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773393.640745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773393.640773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773393.640776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773393.704771:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773393.704864:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773393.704926:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773393.832715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773393.832759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773393.896714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773393.960734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773394.024738:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773394.024746:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773394.024760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773394.088753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773394.152729:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773394.152775:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773394.152786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773394.280730:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773394.280733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773394.280775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773394.280796:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773394.344798:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773394.344802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773394.344809:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773394.344852:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773394.344899:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773394.344923:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773394.472745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773394.536787:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773394.600759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773394.664806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773394.664807:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773394.664839:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773394.664862:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773394.728851:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773394.728871:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773394.728943:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773394.856771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773394.856799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773394.920761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773394.984775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773395.048798:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773395.048810:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773395.048810:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773395.112755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773395.176757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773395.176766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773395.176805:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773395.304781:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773395.304790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773395.304793:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773395.304806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773395.368798:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773395.368817:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773395.368829:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773395.368838:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773395.368881:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773395.368914:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773395.496797:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773395.560751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773395.624768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773395.688769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773395.688785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773395.688807:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773395.688824:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773395.752825:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773395.752847:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773395.752976:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773395.880762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773395.880770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773395.944761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773396.008778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773396.072758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773396.072792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773396.072832:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773396.136755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773396.200823:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773396.200831:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773396.200831:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773396.328800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773396.328804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773396.328852:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773396.328854:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773396.392791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773396.392805:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773396.392843:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773396.392875:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773396.392924:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773396.392944:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773396.520779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773396.584752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773396.648713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773396.712758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773396.712787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773396.712799:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773396.712802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773396.776729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773396.776812:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773396.776971:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773396.904721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773396.904775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773396.968716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773397.032788:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773397.096755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773397.096765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773397.096770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773397.160770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773397.224755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773397.224801:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773397.224808:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773397.352730:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773397.352773:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773397.352788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773397.352803:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773397.416772:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773397.416812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773397.416820:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773397.416821:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773397.416846:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773397.416895:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773397.544755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773397.608729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773397.672734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773397.672778:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773397.672792:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773397.672798:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773397.672811:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2393 00000100:00080000:2.0:1790773397.672824:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773397.672839:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773397.672892:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534667008/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773397.672932:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773397.672938:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773397.672941:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773397.672947:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2393 00000100:00080000:2.0:1790773397.672953:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773397.672964:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773397.672980:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534667008:192.168.204.150@tcp:250:kworker.0 00000008:00100000:2.0:1790773397.672982:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773397.673018:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534667136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773397.673046:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773397.673054:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773397.673058:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773397.673069:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2393 00000100:00080000:2.0:1790773397.673076:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773397.673078:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773397.673094:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773397.673105:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773397.673129:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534667264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773397.673146:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773397.673151:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773397.673156:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773397.673165:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2393 00000100:00100000:1.0:1790773397.673166:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773397/real 1790773397] req@ffff9c4efbcc2680 x1877759534667008/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773397.673175:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773397.673192:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773397.673203:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773397.673220:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534667392/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773397.673229:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534667008/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773397.673246:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773397.673267:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773397.673277:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773397.673288:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2393 00000100:00080000:1.0:1790773397.673291:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773397.673294:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773397.673302:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773397.673308:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773397.673332:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534667008:192.168.204.150@tcp:250:kworker.0 00000008:00100000:2.0:1790773397.673353:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773397.673375:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534667520/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773397.673385:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534667136:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773397.673412:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773397.673418:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773397.673422:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773397.673430:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2393 00000100:00080000:2.0:1790773397.673439:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773397.673447:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773397.673455:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773397.673464:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773397.673471:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773397/real 1790773397] req@ffff9c4efbcc0700 x1877759534667136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773397.673487:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534667136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773397.673498:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534667648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773397.673507:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773397.673521:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773397.673535:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534667136:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773397.673560:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773397.673563:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534667264:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773397.673577:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773397.673584:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773397.673596:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2393 00000100:00080000:2.0:1790773397.673604:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773397.673615:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773397.673621:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773397.673631:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773397.673633:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773397/real 1790773397] req@ffff9c4efbcc0a80 x1877759534667264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773397.673679:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534667264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773397.673687:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773397.673691:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773397.673695:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534667264:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773397.673705:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534667776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773397.673706:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534667392:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773397.673743:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773397.673748:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773397.673751:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773397.673764:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773397.673771:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2393 00000100:00080000:2.0:1790773397.673781:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773397.673790:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773397.673798:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773397.673801:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773397/real 1790773397] req@ffff9c4efbcc2300 x1877759534667392/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773397.673815:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534667904/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773397.673838:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534667392/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773397.673889:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773397.673903:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773397.673906:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773397.673911:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773397.673913:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2393 00000100:00080000:2.0:1790773397.673923:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773397.673933:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773397.673941:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773397.673950:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534667392:192.168.204.150@tcp:8:kworker.0 00000008:00100000:2.0:1790773397.673952:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773397.673984:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534668032/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773397.673989:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534667520:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773397.674029:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773397.674045:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773397/real 1790773397] req@ffff9c4efbcc1880 x1877759534667520/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773397.674064:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534667520/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773397.674072:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773397.674075:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773397.674077:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534667520:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773397.674082:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534667648:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773397.674100:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773397.674112:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773397/real 1790773397] req@ffff9c4efbcc1500 x1877759534667648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773397.674125:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534667648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773397.674151:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773397.674154:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773397.674157:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534667648:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773397.674163:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534667776:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773397.674180:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773397.674201:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773397/real 1790773397] req@ffff9c4efbcc1f80 x1877759534667776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773397.674217:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534667776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773397.674227:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773397.674237:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773397.674241:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534667776:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773397.674247:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534667904:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773397.674266:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773397.674275:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773397/real 1790773397] req@ffff9c4efbcc1c00 x1877759534667904/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773397.674283:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534667904/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773397.674292:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773397.674294:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773397.674297:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534667904:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773397.674327:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534668032:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773397.674345:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773397.674354:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773397/real 1790773397] req@ffff9c4efbcc2a00 x1877759534668032/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773397.674363:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534668032/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773397.674370:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773397.674373:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773397.674375:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534668032:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773397.736756:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773397.736756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773397.736776:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773397.736801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773397.800731:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773397.800743:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773397.800805:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773397.928736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773397.928742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773397.992774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773398.056827:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773398.120738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773398.120786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773398.120786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773398.184751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773398.248712:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773398.248742:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773398.248767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773398.376786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773398.376793:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773398.376842:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773398.376843:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773398.440798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773398.440841:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773398.440857:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773398.440914:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773398.440988:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773398.441020:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773398.568784:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773398.632772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773398.696754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773398.760799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773398.760805:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773398.760808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773398.760811:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773398.824787:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773398.824851:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773398.825068:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773398.952766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773398.952830:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773399.016795:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773399.080816:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773399.144780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773399.144782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773399.144849:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773399.208763:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773399.272756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773399.272759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773399.272779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773399.400772:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773399.400780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773399.400786:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773399.400805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773399.464779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773399.464812:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773399.464821:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773399.464830:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773399.464869:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773399.464876:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773399.592778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773399.656766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773399.720783:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773399.784786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773399.784834:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773399.784853:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773399.784856:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773399.848886:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773399.848952:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773399.849013:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773399.976765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773399.976771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773400.040776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773400.104987:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773400.168783:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773400.168786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773400.168801:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773400.232748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773400.296753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773400.296770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773400.296809:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773400.424767:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773400.424818:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773400.424822:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773400.424822:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773400.488785:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773400.488831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773400.488831:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773400.488845:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773400.488854:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773400.488916:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773400.616755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773400.680782:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773400.744774:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773400.808768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773400.808782:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773400.808786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773400.808812:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773400.872789:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773400.872823:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773400.872963:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773401.000742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773401.000749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773401.064715:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773401.128722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773401.192748:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773401.192756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773401.192756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773401.256764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773401.320757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773401.320770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773401.320785:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773401.448752:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773401.448755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773401.448755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773401.448805:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773401.512744:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773401.512765:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773401.512794:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773401.512800:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773401.512801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773401.512812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773401.640778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773401.704754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773401.768766:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773401.832782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773401.832782:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773401.832794:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773401.832794:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773401.896761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773401.896830:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773401.896900:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773402.024770:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773402.024770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773402.088763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773402.152781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773402.216773:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773402.216843:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773402.216849:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773402.280765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773402.344848:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773402.344852:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773402.344859:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773402.472748:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773402.472764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773402.472802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773402.472802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773402.536742:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773402.536789:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773402.536805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773402.536812:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773402.536812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773402.536859:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773402.664729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773402.728755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773402.792768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773402.792822:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773402.792831:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773402.792837:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773402.792849:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2398 00000100:00080000:0.0:1790773402.792865:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773402.792879:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773402.792930:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534668160/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773402.792969:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773402.792975:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773402.792977:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773402.792983:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2398 00000100:00080000:0.0:1790773402.792988:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773402.792995:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773402.793008:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773402.793034:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534668288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773402.793035:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534668160:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773402.793063:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773402.793068:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773402.793071:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773402.793088:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2398 00000100:00080000:0.0:1790773402.793093:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773402.793101:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773402.793108:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773402.793124:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534668416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773402.793138:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773402.793148:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1790773402.793149:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773402.793152:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773402.793163:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2398 00000100:00080000:0.0:1790773402.793170:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773402.793190:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773402.793200:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773402.793242:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773402/real 1790773402] req@ffff9c4ec7a7d180 x1877759534668160/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773402.793248:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534668544/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773402.793307:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773402.793312:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773402.793315:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773402.793321:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534668160/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773402.793328:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2398 00000100:00080000:0.0:1790773402.793348:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773402.793359:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773402.793361:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773402.793372:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790773402.793374:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773402.793385:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534668160:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773402.793399:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534668672/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773402.793417:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773402.793426:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534668288:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773402.793429:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773402.793441:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773402.793455:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2398 00000800:00000100:1.0:1790773402.793477:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773402.793478:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773402.793507:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773402.793513:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773402/real 1790773402] req@ffff9c4ec7a7c700 x1877759534668288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773402.793523:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773402.793553:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534668800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773402.793561:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534668288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773402.793577:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773402.793593:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773402.793599:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773402.793609:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2398 00000100:00080000:0.0:1790773402.793618:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773402.793619:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773402.793627:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773402.793631:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790773402.793638:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773402.793644:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534668288:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773402.793704:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534668928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773402.793714:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534668416:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773402.793733:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773402.793739:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773402.793743:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773402.793751:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2398 00000100:00080000:0.0:1790773402.793761:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773402.793769:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773402.793776:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773402.793788:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773402.793801:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773402/real 1790773402] req@ffff9c4ec7a7e300 x1877759534668416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773402.793809:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534669056/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773402.793844:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773402.793850:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534668416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773402.793851:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773402.793858:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773402.793869:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2398 00000100:00080000:0.0:1790773402.793879:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773402.793889:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773402.793897:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773402.793899:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773402.793913:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534669184/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773402.793915:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773402.793935:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534668416:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773402.793954:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534668544:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773402.793990:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773402.794007:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773402/real 1790773402] req@ffff9c4ec7a7fb80 x1877759534668544/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773402.794024:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534668544/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773402.794038:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773402.794043:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773402.794048:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534668544:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773402.794055:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534668672:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773402.794075:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773402.794095:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773402/real 1790773402] req@ffff9c4ec7a7ca80 x1877759534668672/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773402.794110:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534668672/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773402.794122:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773402.794126:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773402.794130:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534668672:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773402.794154:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534668800:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773402.794176:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773402.794188:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773402/real 1790773402] req@ffff9c4ec7a7c380 x1877759534668800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773402.794242:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534668800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773402.794266:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773402.794271:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773402.794276:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534668800:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773402.794285:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534668928:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773402.794313:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773402.794328:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773402/real 1790773402] req@ffff9c4ec7a7d880 x1877759534668928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773402.794355:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534668928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773402.794367:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773402.794371:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773402.794376:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534668928:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773402.794383:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534669056:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773402.794402:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773402.794413:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773402/real 1790773402] req@ffff9c4ec7a7ed80 x1877759534669056/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773402.794428:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534669056/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773402.794450:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773402.794455:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773402.794459:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534669056:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773402.794467:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534669184:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773402.794506:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773402.794528:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773402/real 1790773402] req@ffff9c4ec7a7df80 x1877759534669184/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773402.794552:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534669184/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773402.794573:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773402.794581:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773402.794589:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534669184:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773402.856754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773402.856762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773402.856776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773402.856802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773402.920801:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773402.920803:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773402.920853:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773403.048765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773403.048769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773403.112779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773403.176778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773403.240752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773403.240767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773403.240784:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773403.304759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773403.368776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773403.368776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773403.368782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773403.496783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773403.496783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773403.496806:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773403.496807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773403.560722:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773403.560770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773403.560779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773403.560813:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773403.560848:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773403.560850:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773403.688794:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773403.752768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773403.816708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773403.880748:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773403.880772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773403.880791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773403.880796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773403.944759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773403.944828:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773403.944917:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773404.072786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773404.072798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773404.136748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773404.200775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773404.264741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773404.264800:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773404.264801:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773404.328749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773404.392774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773404.392802:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773404.392802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773404.520790:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773404.520811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773404.520812:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773404.520813:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773404.584769:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773404.584791:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773404.584817:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773404.584824:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773404.584842:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773404.584874:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773404.712773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773404.776775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773404.841089:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773404.904774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773404.904777:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773404.904780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773404.904842:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773404.968783:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773404.968868:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773404.968897:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773405.096766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773405.096812:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773405.160770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773405.224789:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773405.288793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773405.288830:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773405.288843:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773405.352715:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773405.416720:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773405.416743:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773405.416744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773405.544828:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773405.544829:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773405.544831:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773405.544853:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773405.608757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773405.608762:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773405.608795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773405.608802:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773405.608834:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773405.608839:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773405.736725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773405.800708:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773405.865733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773405.928781:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773405.928802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773405.928813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773405.928820:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773405.992759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773405.992780:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773405.992860:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773406.120712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773406.120713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773406.184732:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773406.248741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773406.312736:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773406.312742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773406.312742:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773406.376708:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773406.440735:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773406.440736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773406.440736:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773406.568717:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773406.568718:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773406.568722:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773406.568726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773406.632732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773406.632755:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773406.632766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773406.632769:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773406.632802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773406.632818:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773406.760730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773406.824724:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773406.888747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773406.952767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773406.952768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773406.952771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773406.952797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773407.016776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773407.016794:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773407.016841:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773407.144739:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773407.144776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773407.208739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773407.272857:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773407.336752:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773407.336758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773407.336804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773407.400755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773407.464776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773407.464779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773407.464784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773407.592785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773407.592785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773407.592792:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773407.592815:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773407.656768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773407.656800:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773407.656801:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773407.656804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773407.656847:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773407.656859:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773407.784771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773407.848747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773407.912778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773407.912823:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773407.912845:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773407.912854:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773407.912871:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2403 00000100:00080000:3.0:1790773407.912888:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773407.912904:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773407.912951:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534669312/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773407.912989:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773407.912996:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773407.912999:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773407.913005:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2403 00000100:00080000:3.0:1790773407.913012:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773407.913020:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773407.913034:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773407.913046:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534669312:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773407.913059:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534669440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773407.913078:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773407.913083:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773407.913086:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773407.913093:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2403 00000100:00080000:3.0:1790773407.913098:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773407.913107:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773407.913144:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773407.913171:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773407.913173:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534669568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773407.913193:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773407.913199:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773407.913203:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773407.913209:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2403 00000100:00080000:3.0:1790773407.913216:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773407.913225:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773407.913234:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773407.913237:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773407/real 1790773407] req@ffff9c4ef73c0000 x1877759534669312/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773407.913257:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534669696/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773407.913282:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534669312/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773407.913293:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773407.913309:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773407.913310:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773407.913323:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773407.913324:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773407.913340:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534669312:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773407.913345:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2403 00000100:00080000:3.0:1790773407.913367:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773407.913381:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773407.913393:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773407.913406:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534669440:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773407.913420:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534669824/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773407.913438:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773407.913444:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773407.913447:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773407.913454:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2403 00000100:00080000:3.0:1790773407.913462:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773407.913470:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773407.913478:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773407.913488:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773407.913505:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773407/real 1790773407] req@ffff9c4ef73c1500 x1877759534669440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773407.913511:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534669952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773407.913565:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773407.913573:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534669440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773407.913574:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773407.913582:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773407.913597:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2403 00000100:00080000:3.0:1790773407.913609:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773407.913617:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773407.913624:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773407.913626:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773407.913641:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790773407.913686:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534670080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773407.913696:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534669440:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773407.913711:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773407.913723:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773407.913728:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773407.913731:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534669568:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773407.913739:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2403 00000100:00080000:3.0:1790773407.913748:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773407.913755:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773407.913765:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:1.0:1790773407.913784:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773407.913793:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534670208/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773407.913811:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773407/real 1790773407] req@ffff9c4ef73c0700 x1877759534669568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773407.913822:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773407.913837:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773407.913842:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534669568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773407.913846:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773407.913868:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2403 00000100:00080000:1.0:1790773407.913870:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773407.913885:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773407.913885:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773407.913899:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534669568:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773407.913906:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773407.913917:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534669696:192.168.204.150@tcp:8:kworker.0 00000008:00100000:3.0:1790773407.913919:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773407.913953:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534670336/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773407.913965:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773407.913987:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773407/real 1790773407] req@ffff9c4ef73c2a00 x1877759534669696/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773407.914007:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534669696/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773407.914021:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773407.914026:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773407.914031:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534669696:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773407.914066:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534669824:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773407.914089:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773407.914102:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773407/real 1790773407] req@ffff9c4ef73c2d80 x1877759534669824/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773407.914152:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534669824/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773407.914166:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773407.914171:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773407.914175:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534669824:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773407.914184:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534669952:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773407.914211:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773407.914226:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773407/real 1790773407] req@ffff9c4ef73c3800 x1877759534669952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773407.914242:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534669952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773407.914263:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773407.914267:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773407.914272:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534669952:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773407.914280:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534670080:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773407.914300:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773407.914312:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773407/real 1790773407] req@ffff9c4ef73c0a80 x1877759534670080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773407.914327:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534670080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773407.914346:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773407.914353:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773407.914360:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534670080:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773407.914373:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534670208:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773407.914413:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773407.914437:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773407/real 1790773407] req@ffff9c4ef73c0e00 x1877759534670208/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773407.914462:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534670208/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773407.914486:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773407.914490:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773407.914495:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534670208:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773407.914504:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534670336:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773407.914532:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773407.914546:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773407/real 1790773407] req@ffff9c4ef73c2300 x1877759534670336/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773407.914570:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534670336/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773407.914582:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773407.914586:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773407.914590:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534670336:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773407.976767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773407.976795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773407.976803:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773407.976825:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773408.040767:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773408.040805:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773408.040915:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773408.168757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773408.168794:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773408.232745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773408.296757:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773408.360745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773408.360747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773408.360804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773408.424811:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773408.488775:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773408.488796:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773408.488796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773408.616751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773408.616818:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773408.616824:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773408.616836:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773408.680759:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773408.680817:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773408.680823:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773408.680835:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773408.680848:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773408.680886:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773408.808767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773408.872760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773408.936762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773409.000753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773409.000791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773409.000793:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773409.000794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773409.064821:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773409.064822:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773409.064869:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773409.192784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773409.192808:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773409.256750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773409.320782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773409.384750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773409.384751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773409.384764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773409.448775:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773409.512756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773409.512763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773409.512793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773409.640787:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773409.640792:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773409.640821:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773409.640822:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773409.704791:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773409.704809:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773409.704851:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773409.704877:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773409.705021:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773409.705027:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773409.832722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773409.896751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773409.960803:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773410.024800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773410.024804:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773410.024830:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773410.024851:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773410.088788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773410.088796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773410.088844:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773410.216763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773410.216764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773410.280764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773410.344762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773410.408729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773410.408752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773410.408752:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773410.472783:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773410.536745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773410.536746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773410.536783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773410.664767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773410.664788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773410.664792:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773410.664798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773410.728725:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773410.728771:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773410.728772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773410.728774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773410.728775:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773410.728811:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773410.856711:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773410.920723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773410.984729:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773411.048761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773411.048775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773411.048783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773411.048785:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773411.112770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773411.112804:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773411.112806:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773411.240759:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773411.240778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773411.304777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773411.368750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773411.432774:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773411.432805:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773411.432811:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773411.496744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773411.560734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773411.560745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773411.560763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773411.688749:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773411.688771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773411.688779:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773411.688791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773411.752822:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773411.752856:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773411.752857:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773411.752860:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773411.752913:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773411.752916:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773411.880716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773411.944791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773412.008712:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773412.072713:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773412.072725:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773412.072731:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773412.072740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773412.136764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773412.136770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773412.136819:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773412.264743:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773412.264743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773412.328785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773412.392775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773412.456755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773412.456755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773412.456791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773412.520741:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773412.584736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773412.584765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773412.584789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773412.712789:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773412.712830:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773412.712836:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773412.712881:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773412.776763:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773412.776798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773412.776842:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773412.776858:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773412.776859:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773412.776860:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773412.904725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773412.968748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773413.032728:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773413.032743:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773413.032749:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773413.032753:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773413.032762:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2408 00000100:00080000:2.0:1790773413.032771:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773413.032781:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773413.032816:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534670464/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773413.032838:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773413.032841:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773413.032842:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773413.032846:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2408 00000100:00080000:2.0:1790773413.032849:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773413.032853:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773413.032861:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773413.032869:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534670464:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773413.032874:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534670592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773413.032883:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773413.032886:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773413.032888:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773413.032891:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2408 00000100:00080000:2.0:1790773413.032895:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773413.032899:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773413.032903:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773413.032913:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534670720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773413.032920:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773413.032923:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773413.032926:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773413.032927:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773413.032931:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2408 00000100:00080000:2.0:1790773413.032934:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773413.032939:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773413.032943:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773413.032953:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534670848/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773413.032959:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773413.032962:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773413.032964:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773413.032969:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2408 00000100:00100000:1.0:1790773413.032970:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773413/real 1790773413] req@ffff9c4efbcc3100 x1877759534670464/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773413.032972:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773413.032977:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773413.032982:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773413.032995:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534670976/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773413.033003:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534670464/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773413.033051:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773413.033056:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773413.033056:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773413.033059:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773413.033064:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773413.033068:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2408 00000100:00080000:2.0:1790773413.033073:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773413.033074:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534670464:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773413.033080:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773413.033086:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773413.033099:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534671104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773413.033102:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534670592:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773413.033127:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773413.033131:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:1.0:1790773413.033135:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773413.033135:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773413.033145:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2408 00000100:00100000:1.0:1790773413.033151:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773413/real 1790773413] req@ffff9c4efbcc0380 x1877759534670592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773413.033151:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773413.033160:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773413.033168:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773413.033169:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534670592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773413.033183:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534671232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773413.033187:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773413.033195:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773413.033202:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534670592:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773413.033202:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773413.033211:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1790773413.033212:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534670720:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773413.033215:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773413.033222:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2408 00000100:00080000:2.0:1790773413.033230:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773413.033237:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773413.033242:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790773413.033243:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773413.033252:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773413/real 1790773413] req@ffff9c4efbcc2a00 x1877759534670720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773413.033258:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534671360/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773413.033273:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534670720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773413.033273:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773413.033282:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773413.033285:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773413.033292:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2408 00000100:00080000:2.0:1790773413.033298:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773413.033314:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773413.033315:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773413.033321:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773413.033325:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773413.033332:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534671488/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773413.033334:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534670720:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773413.033352:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534670848:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773413.033377:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773413.033386:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773413/real 1790773413] req@ffff9c4efbcc1c00 x1877759534670848/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773413.033395:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534670848/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773413.033403:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773413.033406:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773413.033409:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534670848:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773413.033414:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534670976:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773413.033426:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773413.033432:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773413/real 1790773413] req@ffff9c4efbcc1f80 x1877759534670976/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773413.033440:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534670976/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773413.033446:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773413.033448:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773413.033451:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534670976:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773413.033466:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534671104:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773413.033478:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773413.033484:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773413/real 1790773413] req@ffff9c4efbcc1500 x1877759534671104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773413.033495:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534671104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773413.033508:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773413.033511:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773413.033514:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534671104:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773413.033518:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534671232:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773413.033530:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773413.033536:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773413/real 1790773413] req@ffff9c4efbcc1880 x1877759534671232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773413.033545:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534671232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773413.033550:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773413.033552:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773413.033554:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534671232:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773413.033558:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534671360:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773413.033568:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773413.033574:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773413/real 1790773413] req@ffff9c4efbcc2300 x1877759534671360/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773413.033582:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534671360/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773413.033590:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773413.033593:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773413.033595:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534671360:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773413.033599:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534671488:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773413.033608:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773413.033614:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773413/real 1790773413] req@ffff9c4efbcc0a80 x1877759534671488/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773413.033622:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534671488/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773413.033629:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773413.033631:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773413.033633:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534671488:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773413.096732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773413.096732:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773413.096732:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773413.096740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773413.160717:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773413.160723:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773413.160774:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773413.288707:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773413.288730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773413.352750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773413.416740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773413.480741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773413.480747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773413.480747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773413.544735:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773413.608747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773413.608749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773413.608761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773413.736754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773413.736754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773413.736757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773413.736759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773413.800749:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773413.800785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773413.800787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773413.800789:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773413.800820:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773413.800827:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773413.928710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773413.992751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773414.056726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773414.120746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773414.120752:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773414.120753:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773414.120756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773414.184723:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773414.184805:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773414.184957:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773414.312756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773414.312786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773414.376811:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773414.440922:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773414.504836:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773414.504862:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773414.504862:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773414.568757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773414.632767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773414.632768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773414.632776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773414.760744:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773414.760746:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773414.760830:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773414.760831:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773414.824749:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773414.824780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773414.824827:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773414.824827:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773414.824847:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773414.824869:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773414.952715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773415.016753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773415.080757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773415.144851:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773415.144853:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773415.144854:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773415.144856:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773415.208791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773415.208851:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773415.209065:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773415.336714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773415.336723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773415.400747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773415.464742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773415.528772:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773415.528805:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773415.528811:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773415.592765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773415.656764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773415.656782:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773415.656788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773415.784775:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773415.784775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773415.784775:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773415.784796:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773415.848791:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773415.848836:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773415.848897:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773415.848905:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773415.848917:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773415.848920:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773415.976770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773416.040775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773416.104741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773416.168772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773416.168772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773416.168772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773416.168773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773416.232751:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773416.232768:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773416.232868:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773416.360716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773416.360746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773416.424782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773416.488743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773416.552747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773416.552770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773416.552774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773416.616722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773416.680780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773416.680780:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773416.680833:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773416.808710:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773416.808716:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773416.808721:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773416.808731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773416.872706:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773416.872720:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773416.872732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773416.872770:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773416.872771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773416.872772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773417.000703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773417.064756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773417.128733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773417.192719:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773417.192723:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773417.192740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773417.192742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773417.256800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773417.256834:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773417.256905:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773417.384747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773417.384754:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773417.448783:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773417.512754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773417.576774:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773417.576808:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773417.576838:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773417.640761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773417.704796:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773417.704837:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773417.704878:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773417.832790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773417.832791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773417.832835:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773417.832837:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773417.896894:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773417.896895:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773417.896903:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773417.897150:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773417.897182:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773417.897215:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773418.024765:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773418.088792:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773418.152826:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773418.152835:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773418.152852:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773418.152859:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773418.152875:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2414 00000100:00080000:0.0:1790773418.152890:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773418.152906:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773418.152992:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534671616/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773418.153032:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773418.153038:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773418.153042:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773418.153049:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2414 00000100:00080000:0.0:1790773418.153065:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773418.153077:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773418.153091:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773418.153110:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534671744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773418.153128:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773418.153134:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534671616:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773418.153136:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773418.153139:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773418.153159:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2414 00000100:00080000:0.0:1790773418.153165:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773418.153171:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773418.153179:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773418.153199:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534671872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773418.153222:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773418.153227:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773418.153230:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773418.153249:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2414 00000100:00080000:0.0:1790773418.153263:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773418.153270:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773418.153279:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773418.153280:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773418.153304:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534672000/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773418.153318:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773418.153323:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773418.153326:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773418.153333:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2414 00000100:00080000:0.0:1790773418.153341:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773418.153349:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773418.153355:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773418/real 1790773418] req@ffff9c4ec7a7f480 x1877759534671616/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773418.153362:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773418.153381:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534672128/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773418.153424:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773418.153430:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773418.153433:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773418.153440:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2414 00000100:00080000:0.0:1790773418.153445:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773418.153449:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534671616/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773418.153458:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773418.153478:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773418.153487:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773418.153500:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534672256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773418.153502:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773418.153515:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534671616:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773418.153548:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773418.153557:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773418.153562:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773418.153573:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534671744:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773418.153576:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2414 00000100:00080000:0.0:1790773418.153592:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773418.153610:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773418.153621:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773418.153637:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773418.153698:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773418/real 1790773418] req@ffff9c4ec7a7df80 x1877759534671744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773418.153714:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534672384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773418.153721:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534671744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773418.153802:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773418.153808:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773418.153812:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773418.153822:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2414 00000100:00080000:1.0:1790773418.153825:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773418.153831:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773418.153848:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773418.153848:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790773418.153872:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773418.153872:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534671744:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773418.153899:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534671872:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773418.153905:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534672512/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773418.153966:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773418.153972:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773418.153985:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773418.153986:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773418.154003:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2414 00000100:00080000:0.0:1790773418.154013:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773418.154020:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773418/real 1790773418] req@ffff9c4ec7a7ed80 x1877759534671872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773418.154026:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773418.154049:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773418.154059:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534671872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773418.154074:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534672640/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773418.154146:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773418.154156:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773418.154167:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534671872:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773418.154180:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534672000:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773418.154221:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773418.154238:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773418/real 1790773418] req@ffff9c4ec7a7d880 x1877759534672000/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773418.154251:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534672000/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773418.154261:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773418.154265:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773418.154269:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534672000:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773418.154276:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534672128:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773418.154296:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773418.154305:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773418/real 1790773418] req@ffff9c4ec7a7c380 x1877759534672128/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773418.154317:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534672128/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773418.154326:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773418.154329:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773418.154333:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534672128:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773418.154340:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534672256:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773418.154365:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773418.154381:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773418/real 1790773418] req@ffff9c4ec7a7ca80 x1877759534672256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773418.154400:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534672256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773418.154423:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773418.154429:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773418.154432:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534672256:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773418.154457:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534672384:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773418.154481:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773418.154492:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773418/real 1790773418] req@ffff9c4ec7a7fb80 x1877759534672384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773418.154504:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534672384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773418.154512:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773418.154516:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773418.154519:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534672384:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773418.154525:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534672512:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773418.154540:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773418.154551:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773418/real 1790773418] req@ffff9c4ec7a7e300 x1877759534672512/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773418.154568:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534672512/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773418.154584:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773418.154589:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773418.154595:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534672512:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773418.154605:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534672640:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773418.154634:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773418.154674:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773418/real 1790773418] req@ffff9c4ec7a7c700 x1877759534672640/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773418.154687:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534672640/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773418.154697:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773418.154701:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773418.154704:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534672640:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773418.216767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773418.216769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773418.216807:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773418.216808:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773418.280737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773418.280746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773418.280935:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773418.408712:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773418.408712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773418.472760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773418.536766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773418.600755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773418.600790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773418.600800:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773418.664748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773418.728743:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773418.728743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773418.728752:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773418.856753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773418.856753:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773418.856754:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773418.856778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773418.920747:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773418.920747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773418.920769:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773418.920783:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773418.920796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773418.920814:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773419.048712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773419.112722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773419.176782:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773419.240724:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773419.240724:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773419.240734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773419.240806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773419.304704:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773419.304786:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773419.304946:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773419.432765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773419.432807:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773419.496766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773419.560850:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773419.624762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773419.624771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773419.624804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773419.688724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773419.752720:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773419.752720:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773419.752813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773419.880724:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773419.880727:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773419.880727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773419.880779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773419.944747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773419.944748:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773419.944773:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773419.944795:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773419.944839:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773419.944849:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773420.072737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773420.136805:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773420.200777:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773420.264737:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773420.264737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773420.264737:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773420.264737:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773420.328724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773420.328739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773420.328844:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773420.456752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773420.456752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773420.520703:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773420.584765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773420.648745:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773420.648745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773420.648745:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773420.712727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773420.776779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773420.776779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773420.776779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773420.904734:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773420.904735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773420.904743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773420.904744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773420.968745:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773420.968767:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773420.968772:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773420.968796:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773420.968798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773420.968824:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773421.096801:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773421.160744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773421.224742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773421.288796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773421.288797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773421.288800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773421.288834:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773421.352740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773421.352840:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773421.352845:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773421.480720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773421.480738:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773421.544738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773421.608774:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773421.672737:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773421.672737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773421.672783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773421.736760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773421.800717:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773421.800738:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773421.800762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773421.928812:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773421.928812:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773421.928813:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773421.928813:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773421.992743:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773421.992756:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773421.992774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773421.992796:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773421.992806:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773421.992814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773422.120705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773422.184808:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773422.248742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773422.312758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773422.312774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773422.312779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773422.312797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773422.376757:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773422.376829:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773422.376916:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773422.504709:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773422.504714:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773422.568757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773422.632765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773422.696875:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773422.696877:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773422.696881:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773422.760744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773422.824739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773422.824749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773422.824791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773422.952753:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773422.952761:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773422.952764:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773422.952765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773423.016743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773423.016769:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773423.016770:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773423.016776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773423.016810:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773423.016812:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773423.144718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773423.208718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773423.272705:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773423.272714:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1790773423.272715:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773423.272718:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773423.272728:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2419 00000100:00080000:0.0:1790773423.272738:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773423.272748:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773423.272780:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534672768/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773423.272808:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773423.272811:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773423.272829:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773423.272834:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2419 00000100:00080000:0.0:1790773423.272837:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773423.272842:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773423.272864:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534672768:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773423.272867:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773423.272883:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534672896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773423.272892:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773423.272895:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773423.272896:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773423.272900:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2419 00000100:00080000:0.0:1790773423.272915:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773423.272919:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773423.272922:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773423.272926:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773423.272934:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534673024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773423.272943:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773423.272945:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773423.272947:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773423.272949:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2419 00000100:00080000:0.0:1790773423.272953:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773423.272958:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773423.272962:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773423.272966:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773423/real 1790773423] req@ffff9c4ec7a7d180 x1877759534672768/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773423.272975:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534673152/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773423.272984:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773423.272988:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534672768/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773423.272989:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773423.272992:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773423.272999:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2419 00000100:00080000:0.0:1790773423.273004:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773423.273008:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773423.273009:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773423.273017:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:1.0:1790773423.273019:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773423.273025:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534672768:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773423.273027:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534673280/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773423.273035:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773423.273038:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773423.273039:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773423.273042:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2419 00000100:00080000:0.0:1790773423.273045:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773423.273048:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534672896:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773423.273051:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773423.273058:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773423.273068:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534673408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773423.273076:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773423.273087:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773423.273091:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773423/real 1790773423] req@ffff9c4ec7a7c700 x1877759534672896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773423.273092:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773423.273095:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773423.273102:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2419 00000100:00080000:0.0:1790773423.273107:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773423.273111:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534672896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773423.273113:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773423.273120:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773423.273134:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534673536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773423.273136:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773423.273145:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773423.273147:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773423.273151:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773423.273155:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773423.273155:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534672896:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773423.273163:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2419 00000100:00080000:0.0:1790773423.273165:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773423.273166:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534673024:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773423.273172:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773423.273177:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773423.273188:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534673664/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773423.273191:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773423.273200:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773423.273205:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773423.273205:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773423/real 1790773423] req@ffff9c4ec7a7e300 x1877759534673024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773423.273209:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773423.273217:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2419 00000100:00080000:0.0:1790773423.273223:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773423.273227:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534673024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773423.273231:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773423.273237:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773423.273240:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773423.273245:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773423.273248:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534673792/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773423.273250:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534673024:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773423.273263:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534673152:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773423.273278:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773423.273285:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773423/real 1790773423] req@ffff9c4ec7a7fb80 x1877759534673152/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773423.273294:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534673152/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773423.273301:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773423.273303:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773423.273305:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534673152:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773423.273309:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534673280:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773423.273318:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773423.273324:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773423/real 1790773423] req@ffff9c4ec7a7ca80 x1877759534673280/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773423.273331:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534673280/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773423.273342:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773423.273345:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773423.273347:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534673280:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773423.273357:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534673408:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773423.273367:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773423.273373:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773423/real 1790773423] req@ffff9c4ec7a7c380 x1877759534673408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773423.273381:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534673408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773423.273400:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773423.273403:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773423.273405:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534673408:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773423.273409:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534673536:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773423.273421:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773423.273428:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773423/real 1790773423] req@ffff9c4ec7a7d880 x1877759534673536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773423.273436:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534673536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773423.273441:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773423.273444:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773423.273447:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534673536:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773423.273451:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534673664:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773423.273461:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773423.273467:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773423/real 1790773423] req@ffff9c4ec7a7ed80 x1877759534673664/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773423.273474:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534673664/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773423.273481:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773423.273483:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773423.273485:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534673664:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773423.273489:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534673792:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773423.273499:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773423.273504:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773423/real 1790773423] req@ffff9c4ec7a7df80 x1877759534673792/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773423.273512:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534673792/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773423.273518:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773423.273520:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773423.273522:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534673792:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773423.336727:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773423.336743:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773423.336751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773423.336753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773423.400735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773423.400744:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773423.400829:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773423.528726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773423.528746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773423.592726:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773423.656848:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773423.720739:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773423.720757:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773423.720792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773423.784770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773423.848783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773423.848794:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773423.848799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773423.976779:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773423.976779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773423.976780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773423.976794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773424.040749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773424.040750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773424.040772:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773424.040776:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773424.040831:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773424.040864:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773424.168784:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773424.232732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773424.296737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773424.360750:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773424.360760:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773424.360762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773424.360787:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773424.424758:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773424.424853:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773424.424970:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773424.552816:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773424.552824:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773424.616857:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773424.680818:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773424.744751:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773424.744751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773424.744768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773424.808713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773424.872729:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773424.872742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773424.872805:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773425.000725:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773425.000733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773425.000818:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773425.000818:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773425.064771:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773425.064830:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773425.064854:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773425.064882:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773425.064885:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773425.064923:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773425.192742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773425.256724:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773425.320737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773425.384766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773425.384826:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773425.384834:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773425.384843:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773425.448858:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773425.448876:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773425.448908:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773425.576758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773425.576814:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773425.640758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773425.704825:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773425.768749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773425.768846:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773425.768865:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773425.832743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773425.896821:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773425.896823:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773425.896826:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773426.024755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773426.024755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773426.024757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773426.024848:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773426.088738:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773426.088738:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773426.088740:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773426.088777:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773426.088792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773426.088846:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773426.216749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773426.280751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773426.344819:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773426.408825:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773426.408834:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773426.408888:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773426.408910:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773426.472862:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773426.472879:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773426.472966:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773426.600855:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773426.600867:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773426.664848:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773426.728838:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773426.792719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773426.792782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773426.792794:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773426.856719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773426.920806:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773426.920808:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773426.920825:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773427.048764:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773427.048764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773427.048764:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773427.048768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773427.112705:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773427.112735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773427.112750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773427.112771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773427.112772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773427.112777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773427.240726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773427.304698:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773427.368796:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773427.432848:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773427.432852:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773427.432855:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773427.432855:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773427.496782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773427.496806:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773427.496903:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773427.624745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773427.624758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773427.688800:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773427.752806:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773427.816799:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773427.816801:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773427.816831:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773427.880854:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773427.944794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773427.944800:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773427.944826:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773428.072801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773428.072806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773428.072839:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773428.072854:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773428.136763:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773428.136793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773428.136793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773428.136824:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773428.136843:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773428.136864:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773428.264759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773428.328792:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773428.392770:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773428.392771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773428.392783:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773428.392790:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773428.392814:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2424 00000100:00080000:0.0:1790773428.392831:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773428.392846:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773428.392889:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534673920/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773428.392918:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773428.392922:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773428.392924:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773428.392928:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2424 00000100:00080000:0.0:1790773428.392933:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773428.392938:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773428.392965:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773428.392977:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534674048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773428.392984:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534673920:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773428.392990:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773428.392995:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773428.392996:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773428.393001:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2424 00000100:00080000:0.0:1790773428.393005:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773428.393009:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773428.393013:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773428.393023:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534674176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773428.393030:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773428.393033:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773428.393034:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773428.393037:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2424 00000100:00080000:0.0:1790773428.393040:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773428.393043:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773428.393047:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:1.0:1790773428.393053:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773428.393057:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534674304/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773428.393065:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773428.393068:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773428.393070:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773428.393074:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2424 00000100:00080000:0.0:1790773428.393080:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773428.393083:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773428.393087:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773428.393096:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534674432/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773428.393105:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773428.393106:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773428/real 1790773428] req@ffff9c4ec7a7f480 x1877759534673920/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773428.393110:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773428.393114:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773428.393123:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2424 00000100:00080000:0.0:1790773428.393127:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773428.393132:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773428.393136:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773428.393136:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534673920/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773428.393148:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773428.393153:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773428.393157:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534674560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773428.393158:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534673920:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773428.393172:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773428.393175:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773428.393177:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773428.393181:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2424 00000100:00080000:0.0:1790773428.393183:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773428.393187:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773428.393191:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773428.393192:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534674048:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773428.393205:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534674688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773428.393218:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773428.393222:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773428.393226:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773428.393227:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773428.393236:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2424 00000100:00080000:0.0:1790773428.393243:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773428.393247:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773428/real 1790773428] req@ffff9c4ec7a7df80 x1877759534674048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773428.393253:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773428.393261:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773428.393263:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534674048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773428.393275:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534674816/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773428.393286:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773428.393287:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773428.393293:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773428.393297:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534674048:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773428.393298:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773428.393302:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773428.393308:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2424 00000100:00100000:1.0:1790773428.393312:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534674176:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773428.393314:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773428.393321:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773428.393327:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773428.393338:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534674944/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773428.393351:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773428.393367:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773428/real 1790773428] req@ffff9c4ec7a7ed80 x1877759534674176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773428.393378:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534674176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773428.393385:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773428.393388:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773428.393391:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534674176:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773428.393396:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534674304:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773428.393430:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773428.393440:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773428/real 1790773428] req@ffff9c4ec7a7d880 x1877759534674304/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773428.393452:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534674304/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773428.393461:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773428.393464:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773428.393469:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534674304:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773428.393475:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534674432:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773428.393491:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773428.393499:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773428/real 1790773428] req@ffff9c4ec7a7c380 x1877759534674432/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773428.393510:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534674432/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773428.393519:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773428.393522:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773428.393525:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534674432:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773428.393548:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534674560:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773428.393568:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773428.393579:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773428/real 1790773428] req@ffff9c4ec7a7ca80 x1877759534674560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773428.393593:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534674560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773428.393611:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773428.393615:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773428.393619:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534674560:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773428.393662:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534674688:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773428.393684:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773428.393769:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773428/real 1790773428] req@ffff9c4ec7a7fb80 x1877759534674688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773428.393784:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534674688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773428.393793:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773428.393796:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773428.393800:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534674688:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773428.393806:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534674816:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773428.393828:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773428.393853:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773428/real 1790773428] req@ffff9c4ec7a7e300 x1877759534674816/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773428.393861:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534674816/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773428.393869:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773428.393871:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773428.393873:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534674816:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773428.393877:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534674944:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773428.393891:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773428.393903:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773428/real 1790773428] req@ffff9c4ec7a7c700 x1877759534674944/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773428.393916:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534674944/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773428.393929:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773428.393933:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773428.393937:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534674944:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773428.456772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773428.456791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773428.456791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773428.456809:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773428.520840:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773428.520860:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773428.520921:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773428.648730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773428.648743:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773428.712797:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773428.776773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773428.840721:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773428.840771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773428.840790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773428.904770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773428.968805:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773428.968814:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773428.968830:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773429.096807:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773429.096817:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773429.096825:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773429.096861:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773429.160825:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773429.160831:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773429.160840:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773429.160896:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773429.160920:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773429.160928:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773429.288796:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773429.352774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773429.416837:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773429.480747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773429.480749:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773429.480750:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773429.480754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773429.544746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773429.544790:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773429.544825:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773429.672772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773429.672795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773429.736765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773429.800758:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773429.864772:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773429.864774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773429.864779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773429.928781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773429.992793:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773429.992803:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773429.992822:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773430.120800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773430.120810:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773430.120829:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773430.120831:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773430.184713:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773430.184729:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773430.184741:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773430.184773:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773430.184810:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773430.184816:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773430.312736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773430.376756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773430.440765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773430.504787:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773430.504787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773430.504787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773430.504788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773430.568740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773430.568744:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773430.568817:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773430.696766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773430.696766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773430.760782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773430.824755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773430.888804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773430.888804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773430.888804:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773430.952786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773431.016802:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773431.016803:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773431.016817:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773431.144835:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773431.144836:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773431.144836:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773431.144837:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773431.208784:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773431.208798:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773431.208811:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773431.208815:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773431.208833:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773431.208841:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773431.336725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773431.400797:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773431.464769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773431.528779:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773431.528785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773431.528787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773431.528791:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773431.592732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773431.592778:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773431.592864:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773431.720802:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773431.720831:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773431.784762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773431.848755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773431.912778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773431.912783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773431.912823:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773431.976778:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773432.040740:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773432.040789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773432.040811:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773432.168783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773432.168783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773432.168809:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773432.168810:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773432.232781:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773432.232788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773432.232816:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773432.232818:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773432.232855:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773432.232868:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773432.360797:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773432.424775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773432.488750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773432.552808:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773432.552808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773432.552810:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773432.552832:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773432.616794:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773432.616901:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773432.616934:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773432.744757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773432.744758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773432.808753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773432.872775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773432.936741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773432.936748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773432.936750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773433.000752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773433.064752:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773433.064753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773433.064754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773433.192762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773433.192764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773433.192766:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773433.192766:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773433.256747:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773433.256760:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773433.256768:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773433.256782:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773433.256818:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773433.256824:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773433.384710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773433.448742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773433.512734:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773433.512739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773433.512744:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773433.512748:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773433.512759:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2429 00000100:00080000:1.0:1790773433.512769:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773433.512781:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773433.512815:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534675072/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773433.512838:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773433.512841:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773433.512843:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773433.512846:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2429 00000100:00080000:1.0:1790773433.512850:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773433.512855:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773433.512864:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773433.512870:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534675072:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773433.512882:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534675200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773433.512896:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773433.512900:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773433.512903:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773433.512909:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2429 00000100:00080000:1.0:1790773433.512922:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773433.512931:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773433.512931:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773433.512941:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773433.512958:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534675328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773433.512970:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773433.512971:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773433/real 1790773433] req@ffff9c4efac4e300 x1877759534675072/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773433.512982:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773433.512987:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773433.512993:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2429 00000100:00100000:0.0:1790773433.512994:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534675072/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773433.512998:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773433.513007:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773433.513010:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773433.513012:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00080000:0.0:1790773433.513016:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773433.513021:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534675072:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773433.513028:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534675456/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773433.513043:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534675200:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773433.513044:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773433.513052:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773433.513054:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773433.513058:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2429 00000100:00080000:1.0:1790773433.513065:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773433.513069:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773433.513072:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773433.513077:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773433.513083:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773433/real 1790773433] req@ffff9c4efac4ea00 x1877759534675200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773433.513088:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534675584/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773433.513100:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534675200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773433.513101:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773433.513110:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773433.513114:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773433.513121:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2429 00000100:00080000:1.0:1790773433.513125:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773433.513130:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773433.513130:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773433.513133:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773433.513137:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773433.513142:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534675200:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773433.513145:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534675712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773433.513151:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534675328:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773433.513159:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773433.513163:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773433.513164:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773433.513169:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2429 00000100:00080000:1.0:1790773433.513173:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773433.513174:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773433.513181:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773433.513186:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773433.513187:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773433/real 1790773433] req@ffff9c4efac4e680 x1877759534675328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773433.513197:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534675840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773433.513200:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534675328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773433.513208:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773433.513213:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773433.513219:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534675328:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773433.513223:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773433.513227:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773433.513229:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773433.513233:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534675456:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773433.513234:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2429 00000100:00080000:1.0:1790773433.513240:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773433.513244:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773433.513247:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773433.513260:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534675968/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773433.513268:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773433.513269:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773433.513274:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773433.513276:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773433.513281:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773433/real 1790773433] req@ffff9c4efac4f480 x1877759534675456/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773433.513282:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2429 00000100:00080000:1.0:1790773433.513287:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773433.513294:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773433.513300:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773433.513302:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534675456/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773433.513311:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534676096/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773433.513312:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773433.513320:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773433.513327:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534675456:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773433.513339:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534675584:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773433.513355:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773433.513374:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773433/real 1790773433] req@ffff9c4efac4f800 x1877759534675584/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773433.513382:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534675584/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773433.513389:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773433.513391:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773433.513393:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534675584:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773433.513397:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534675712:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773433.513407:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773433.513413:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773433/real 1790773433] req@ffff9c4efac4c700 x1877759534675712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773433.513421:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534675712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773433.513433:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773433.513436:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773433.513438:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534675712:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773433.513442:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534675840:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773433.513452:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773433.513459:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773433/real 1790773433] req@ffff9c4efac4ed80 x1877759534675840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773433.513467:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534675840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773433.513472:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773433.513474:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773433.513476:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534675840:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773433.513487:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534675968:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773433.513498:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773433.513504:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773433/real 1790773433] req@ffff9c4efac4dc00 x1877759534675968/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773433.513512:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534675968/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773433.513519:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773433.513520:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773433.513523:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534675968:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773433.513528:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534676096:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773433.513538:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773433.513543:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773433/real 1790773433] req@ffff9c4efac4df80 x1877759534676096/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773433.513551:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534676096/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773433.513558:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773433.513560:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773433.513562:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534676096:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773433.576714:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773433.576722:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773433.576723:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773433.576725:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773433.640738:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773433.640746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773433.640780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773433.768713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773433.768723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773433.832734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773433.896709:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773433.960706:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773433.960720:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773433.960724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773434.024705:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773434.088718:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773434.088718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773434.088721:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773434.216734:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773434.216740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773434.216741:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773434.216746:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773434.280719:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773434.280720:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773434.280731:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773434.280734:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773434.280761:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773434.280763:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773434.408713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773434.472731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773434.536711:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773434.600724:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773434.600730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773434.600746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773434.600769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773434.664720:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773434.664770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773434.664831:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773434.792769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773434.792775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773434.856739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773434.920763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773434.984715:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773434.984775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773434.984794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773435.048729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773435.112724:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773435.112732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773435.112790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773435.240746:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773435.240750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773435.240776:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773435.240824:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773435.304748:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773435.304753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773435.304793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773435.304822:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773435.304834:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773435.304893:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773435.433706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773435.496744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773435.560752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773435.624773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773435.624773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773435.624774:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773435.624845:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773435.688798:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773435.688820:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773435.689033:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773435.816733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773435.816737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773435.880802:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773435.944723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773436.008759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773436.008785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773436.008796:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773436.072740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773436.136830:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773436.136837:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773436.136842:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773436.264772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773436.264783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773436.264795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773436.264910:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773436.328771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773436.328787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773436.328805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773436.328808:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773436.328847:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773436.328874:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773436.456755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773436.520743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773436.584767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773436.648807:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773436.648818:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773436.648827:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773436.648827:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773436.712773:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773436.712925:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773436.713193:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773436.840765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773436.840781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773436.904840:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773436.968749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773437.032760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773437.032776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773437.032777:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773437.096794:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773437.160768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773437.160808:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773437.160811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773437.288838:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773437.288853:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773437.288864:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773437.288870:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773437.352775:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773437.352793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773437.352820:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773437.352827:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773437.352851:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773437.352877:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773437.480771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773437.544765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773437.608780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773437.672756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773437.672762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773437.672798:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773437.672800:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773437.736824:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773437.736922:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773437.736931:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773437.864768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773437.864783:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773437.928758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773437.992793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773438.056765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773438.056766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773438.056809:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773438.120782:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773438.184779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773438.184788:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773438.184791:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773438.312779:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773438.312788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773438.312802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773438.312811:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773438.376812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773438.376823:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773438.376823:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773438.376845:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773438.376889:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773438.376911:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773438.504707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773438.568768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773438.632757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773438.632794:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773438.632805:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773438.632810:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773438.632822:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2434 00000100:00080000:0.0:1790773438.632851:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773438.632867:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773438.632918:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534676224/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773438.632964:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773438.632971:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773438.632975:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773438.632983:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2434 00000100:00080000:0.0:1790773438.632989:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773438.632998:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773438.633011:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773438.633011:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534676224:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773438.633060:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534676352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773438.633076:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773438.633081:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773438.633083:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773438.633089:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2434 00000100:00080000:0.0:1790773438.633094:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773438.633137:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773438.633145:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773438.633148:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773438.633177:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534676480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773438.633208:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773438.633211:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773438/real 1790773438] req@ffff9c4ec7a7d180 x1877759534676224/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773438.633223:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773438.633235:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773438.633250:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534676224/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773438.633251:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2434 00000100:00080000:0.0:1790773438.633274:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773438.633278:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773438.633289:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773438.633296:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773438.633298:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534676224:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773438.633325:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773438.633345:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534676352:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773438.633376:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534676608/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773438.633390:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773438.633401:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773438.633420:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773438.633438:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773438.633447:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2434 00000100:00080000:0.0:1790773438.633453:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773438.633464:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773438.633486:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773438.633487:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773438/real 1790773438] req@ffff9c4ec7a7c700 x1877759534676352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773438.633504:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534676736/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773438.633532:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534676352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773438.633545:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773438.633568:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773438.633569:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773438.633573:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773438.633579:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773438.633590:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2434 00000100:00100000:1.0:1790773438.633591:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534676352:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773438.633603:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773438.633611:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534676480:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773438.633632:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773438.633642:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773438.633665:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534676864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773438.633680:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773438.633707:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773438/real 1790773438] req@ffff9c4ec7a7e300 x1877759534676480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773438.633718:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534676480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773438.633726:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773438.633729:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773438.633733:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534676480:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773438.633749:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773438.633755:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534676608:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773438.633773:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773438.633780:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773438.633788:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773438.633789:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2434 00000100:00080000:0.0:1790773438.633800:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773438.633802:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773438/real 1790773438] req@ffff9c4ec7a7fb80 x1877759534676608/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773438.633815:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773438.633847:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534676608/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773438.633853:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773438.633860:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773438.633867:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773438.633870:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534676608:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773438.633880:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534676992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773438.633882:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534676736:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773438.633913:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773438.633923:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1790773438.633924:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773438.633930:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773438.633942:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2434 00000100:00100000:1.0:1790773438.633944:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773438/real 1790773438] req@ffff9c4ec7a7ca80 x1877759534676736/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773438.633955:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773438.633978:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773438.633980:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534676736/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773438.633991:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773438.634011:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773438.634021:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773438.634026:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534677120/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773438.634034:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534676736:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773438.634066:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534676864:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773438.634077:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773438.634083:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773438.634088:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773438.634095:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2434 00000800:00000100:1.0:1790773438.634101:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773438.634102:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773438.634116:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773438.634133:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773438/real 1790773438] req@ffff9c4ec7a7c380 x1877759534676864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773438.634143:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773438.634149:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534676864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773438.634159:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534677248/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773438.634165:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773438.634178:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773438.634184:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534676864:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773438.634206:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534676992:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773438.634222:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773438.634230:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773438/real 1790773438] req@ffff9c4ec7a7d880 x1877759534676992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773438.634238:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534676992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773438.634246:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773438.634249:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773438.634252:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534676992:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773438.634256:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534677120:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773438.634268:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773438.634274:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773438/real 1790773438] req@ffff9c4ec7a7ed80 x1877759534677120/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773438.634282:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534677120/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773438.634291:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773438.634293:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773438.634296:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534677120:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773438.634301:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534677248:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773438.634312:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773438.634318:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773438/real 1790773438] req@ffff9c4ec7a7df80 x1877759534677248/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773438.634326:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534677248/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773438.634334:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773438.634336:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773438.634339:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534677248:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773438.696812:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773438.696812:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773438.696834:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773438.696865:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773438.760850:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773438.760941:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773438.761004:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773438.888760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773438.888772:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773438.952761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773439.016755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773439.080708:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773439.080709:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773439.080712:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773439.144735:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773439.208770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773439.208770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773439.208770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773439.336760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773439.336770:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773439.336770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773439.336801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773439.400746:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773439.400750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773439.400756:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773439.400786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773439.400826:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773439.400827:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773439.528779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773439.592735:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773439.656784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773439.720784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773439.720799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773439.720803:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773439.720803:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773439.784749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773439.784765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773439.784806:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773439.912703:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773439.912731:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773439.976772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773440.040777:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773440.104810:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773440.104821:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773440.104824:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773440.168723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773440.232723:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773440.232746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773440.232746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773440.360728:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773440.360743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773440.360748:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773440.360749:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773440.424787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773440.424787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773440.424793:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773440.424811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773440.424852:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773440.424872:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773440.552739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773440.616742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773440.680761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773440.744770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773440.744775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773440.744779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773440.744812:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773440.808759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773440.808808:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773440.808883:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773440.936771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773440.936792:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773441.000776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773441.064748:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773441.128755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773441.128755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773441.128764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773441.192736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773441.256778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773441.256778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773441.256795:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773441.384757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773441.384765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773441.384766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773441.384810:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773441.448739:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773441.448749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773441.448773:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773441.448830:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773441.448833:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773441.448841:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773441.576727:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773441.640729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773441.704732:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773441.768711:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773441.768725:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773441.768727:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773441.768732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773441.832763:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773441.832785:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773441.832923:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773441.960771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773441.960788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773442.024755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773442.088764:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773442.152786:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773442.152803:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773442.152808:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773442.216764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773442.280792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773442.280808:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773442.280812:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773442.408737:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773442.408755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773442.408756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773442.408817:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773442.472789:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773442.472801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773442.472876:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773442.472884:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773442.472885:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773442.472936:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773442.600774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773442.664840:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773442.728746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773442.792742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773442.792746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773442.792759:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773442.792788:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773442.856766:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773442.856839:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773442.856999:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773442.984766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773442.984776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773443.048804:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773443.112744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773443.176810:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773443.176817:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773443.176843:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773443.240747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773443.304780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773443.304802:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773443.304822:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773443.432795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773443.432808:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773443.432813:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773443.432818:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773443.496766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773443.496780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773443.496780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773443.496813:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773443.496841:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773443.496862:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773443.624742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773443.688774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773443.752742:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773443.752753:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773443.752759:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773443.752771:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2439 00000100:00100000:1.0:1790773443.752780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773443.752783:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773443.752799:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773443.752840:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534677376/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773443.752873:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773443.752878:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773443.752880:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773443.752885:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2439 00000100:00080000:0.0:1790773443.752889:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773443.752894:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773443.752905:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773443.752917:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534677376:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773443.752923:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534677504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773443.752937:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773443.752941:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773443.752943:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773443.752948:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2439 00000100:00080000:0.0:1790773443.752951:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773443.752956:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773443.752963:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773443.752974:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534677632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773443.752985:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773443.752989:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773443.752991:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773443.752999:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773443.753001:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2439 00000100:00080000:0.0:1790773443.753008:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773443.753015:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773443.753021:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773443.753035:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534677760/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773443.753046:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773443.753050:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773443.753055:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773443.753057:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773443/real 1790773443] req@ffff9c4ec7a7f480 x1877759534677376/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773443.753062:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2439 00000100:00080000:0.0:1790773443.753069:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773443.753078:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773443.753086:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773443.753102:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534677888/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773443.753114:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534677376/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773443.753117:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773443.753130:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773443.753137:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773443.753150:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2439 00000100:00080000:1.0:1790773443.753152:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773443.753160:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773443.753163:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773443.753172:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534677376:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773443.753177:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773443.753187:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773443.753203:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534677504:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773443.753210:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534678016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773443.753232:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773443.753241:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:1.0:1790773443.753242:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773443.753250:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773443.753263:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2439 00000100:00100000:1.0:1790773443.753265:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773443/real 1790773443] req@ffff9c4ec7a7df80 x1877759534677504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773443.753274:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773443.753287:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534677504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773443.753291:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773443.753302:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773443.753324:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534678144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773443.753329:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773443.753341:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773443.753342:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773443.753352:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1790773443.753354:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534677504:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773443.753357:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773443.753369:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2439 00000100:00100000:1.0:1790773443.753371:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534677632:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773443.753375:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773443.753397:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773443.753404:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:1.0:1790773443.753408:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773443.753446:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534678272/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773443.753473:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773443/real 1790773443] req@ffff9c4ec7a7ed80 x1877759534677632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773443.753485:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773443.753498:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773443.753504:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773443.753514:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2439 00000100:00100000:1.0:1790773443.753521:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534677632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773443.753524:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773443.753539:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773443.753545:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790773443.753549:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773443.753557:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773443.753567:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534677632:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773443.753574:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534678400/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773443.753601:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534677760:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773443.753635:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773443.753651:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773443/real 1790773443] req@ffff9c4ec7a7d880 x1877759534677760/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773443.753666:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534677760/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773443.753677:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773443.753712:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773443.753717:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534677760:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773443.753724:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534677888:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773443.753747:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773443.753758:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773443/real 1790773443] req@ffff9c4ec7a7c380 x1877759534677888/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773443.753770:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534677888/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773443.753780:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773443.753784:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773443.753787:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534677888:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773443.753805:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534678016:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773443.753822:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773443.753831:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773443/real 1790773443] req@ffff9c4ec7a7ca80 x1877759534678016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773443.753843:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534678016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773443.753858:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773443.753861:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773443.753865:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534678016:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773443.753870:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534678144:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773443.753884:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773443.753895:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773443/real 1790773443] req@ffff9c4ec7a7fb80 x1877759534678144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773443.753906:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534678144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773443.753915:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773443.753918:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773443.753921:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534678144:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773443.753926:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534678272:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773443.753940:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773443.753948:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773443/real 1790773443] req@ffff9c4ec7a7e300 x1877759534678272/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773443.753959:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534678272/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773443.753970:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773443.753973:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773443.753977:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534678272:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773443.753982:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534678400:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773443.753996:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773443.754004:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773443/real 1790773443] req@ffff9c4ec7a7c700 x1877759534678400/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773443.754014:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534678400/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773443.754025:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773443.754029:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773443.754033:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534678400:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773443.816731:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773443.816742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773443.816759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773443.816799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773443.880803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773443.880821:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773443.880856:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773444.008728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773444.008746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773444.072845:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773444.136749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773444.200751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773444.200760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773444.200830:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773444.264731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773444.328790:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773444.328816:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773444.328817:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773444.456764:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773444.456789:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773444.456790:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773444.456799:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773444.520791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773444.520799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773444.520846:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773444.520853:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773444.520871:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773444.520898:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773444.648756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773444.712751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773444.776726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773444.840756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773444.840763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773444.840794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773444.840803:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773444.904813:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773444.904880:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773444.904889:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773445.032765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773445.032788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773445.096724:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773445.160806:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773445.224779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773445.224791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773445.224841:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773445.288758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773445.352783:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773445.352814:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773445.352834:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773445.480762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773445.480808:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773445.480814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773445.480815:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773445.544776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773445.544823:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773445.544827:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773445.544827:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773445.544881:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773445.544885:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773445.672746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773445.736795:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773445.800784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773445.864804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773445.864838:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773445.864841:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773445.864844:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773445.928860:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773445.928883:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773445.929015:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773446.056779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773446.056779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773446.120750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773446.184790:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773446.248837:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773446.248843:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773446.248843:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773446.312784:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773446.376773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773446.376773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773446.376814:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773446.504779:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773446.504779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773446.504780:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773446.504780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773446.568736:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773446.568736:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773446.568742:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773446.568742:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773446.568771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773446.568771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773446.696734:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773446.760709:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773446.824723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773446.888751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773446.888752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773446.888754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773446.888756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773446.952728:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773446.952758:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773446.952810:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773447.080706:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773447.080742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773447.144716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773447.208706:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773447.272747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773447.272749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773447.272752:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773447.336704:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773447.400758:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773447.400758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773447.400764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773447.528747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773447.528754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773447.528755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773447.528776:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773447.592763:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773447.592763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773447.592764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773447.592774:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773447.592822:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773447.592823:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773447.720747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773447.784711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773447.848698:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773447.912709:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773447.912716:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773447.912718:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773447.912722:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773447.976699:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773447.976727:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773447.976756:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773448.104706:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773448.104720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773448.168723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773448.232725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773448.296778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773448.296778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773448.296780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773448.360772:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773448.424773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773448.424820:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773448.424840:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773448.552760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773448.552761:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773448.552761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773448.552761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773448.616772:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773448.616772:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773448.616795:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773448.616813:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773448.616849:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773448.616863:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773448.744762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773448.808761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773448.872777:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773448.872788:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773448.872795:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790773448.872802:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773448.872812:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2444 00000100:00080000:1.0:1790773448.872827:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773448.872857:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773448.872920:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534678528/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773448.872962:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773448.872966:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773448.872968:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773448.872972:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2444 00000100:00080000:1.0:1790773448.872986:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773448.872992:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773448.873006:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773448.873023:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534678656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773448.873034:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773448.873052:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534678528:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773448.873061:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773448.873067:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773448.873077:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2444 00000100:00080000:1.0:1790773448.873081:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773448.873086:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773448.873105:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773448.873120:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534678784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773448.873131:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773448.873135:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773448.873138:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773448.873142:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2444 00000100:00080000:1.0:1790773448.873190:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773448.873195:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773448.873200:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:0.0:1790773448.873211:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773448.873235:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534678912/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773448.873249:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773448.873253:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773448.873256:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773448.873261:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2444 00000100:00080000:1.0:1790773448.873265:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773448.873269:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773448.873275:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773448.873293:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534679040/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773448.873307:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773448/real 1790773448] req@ffff9c4efac4fb80 x1877759534678528/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773448.873331:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773448.873352:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773448.873358:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773448.873371:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2444 00000100:00080000:1.0:1790773448.873379:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773448.873385:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534678528/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773448.873409:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773448.873419:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773448.873421:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773448.873435:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773448.873442:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534678528:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773448.873449:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534679168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773448.873462:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773448.873468:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773448.873472:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773448.873478:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534678656:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773448.873480:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2444 00000100:00080000:1.0:1790773448.873508:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773448.873515:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773448.873521:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773448.873537:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534679296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773448.873539:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773448.873561:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773448.873564:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773448.873566:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773448.873580:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2444 00000100:00080000:1.0:1790773448.873583:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773448.873587:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773448.873591:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773448.873601:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534679424/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773448.873612:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773448/real 1790773448] req@ffff9c4efac4df80 x1877759534678656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773448.873614:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773448.873627:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773448.873631:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773448.873640:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2444 00000100:00080000:1.0:1790773448.873648:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773448.873653:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773448.873658:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773448.873670:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534679552/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773448.873707:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534678656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773448.873736:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773448.873742:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773448.873746:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534678656:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773448.873755:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534678784:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773448.873808:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773448.873833:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773448/real 1790773448] req@ffff9c4efac4dc00 x1877759534678784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773448.873861:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534678784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773448.873881:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773448.873889:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773448.873896:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534678784:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773448.873906:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534678912:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773448.873940:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773448.873956:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773448/real 1790773448] req@ffff9c4efac4ed80 x1877759534678912/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773448.873990:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534678912/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773448.874008:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773448.874012:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773448.874015:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534678912:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773448.874022:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534679040:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773448.874045:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773448.874060:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773448/real 1790773448] req@ffff9c4efac4c700 x1877759534679040/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773448.874092:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534679040/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773448.874106:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773448.874111:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773448.874116:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534679040:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773448.874126:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534679168:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773448.874147:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773448.874187:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773448/real 1790773448] req@ffff9c4efac4f800 x1877759534679168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773448.874203:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534679168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773448.874222:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773448.874227:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773448.874233:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534679168:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773448.874262:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534679296:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773448.874300:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773448.874362:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773448/real 1790773448] req@ffff9c4efac4f480 x1877759534679296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773448.874391:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534679296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773448.874402:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773448.874406:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773448.874410:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534679296:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773448.874416:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534679424:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773448.874437:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773448.874448:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773448/real 1790773448] req@ffff9c4efac4e680 x1877759534679424/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773448.874458:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534679424/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773448.874468:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773448.874471:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773448.874474:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534679424:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773448.874480:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534679552:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773448.874504:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773448.874513:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773448/real 1790773448] req@ffff9c4efac4ea00 x1877759534679552/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773448.874527:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534679552/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773448.874540:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773448.874545:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773448.874550:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534679552:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773448.936769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773448.936769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773448.936804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773448.936805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773449.000739:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773449.000813:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773449.000966:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773449.128713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773449.128716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773449.192767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773449.256742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773449.320751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773449.320759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773449.320764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773449.384726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773449.448805:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773449.448805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773449.448811:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773449.576744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773449.576744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773449.576745:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773449.576746:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773449.640764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773449.640771:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773449.640793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773449.640799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773449.640827:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773449.640845:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773449.768732:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773449.832748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773449.896735:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773449.960713:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773449.960735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773449.960735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773449.960819:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773450.024736:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773450.024737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773450.024805:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773450.152752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773450.152755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773450.216727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773450.280714:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773450.344714:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773450.344718:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773450.344720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773450.408713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773450.472754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773450.472754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773450.472754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773450.600763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773450.600766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773450.600787:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773450.600797:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773450.664719:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773450.664719:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773450.664732:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773450.664732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773450.664760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773450.664761:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773450.792716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773450.856756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773450.920788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773450.984749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773450.984750:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773450.984776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773450.984778:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773451.048753:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773451.048756:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773451.048831:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773451.176758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773451.176759:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773451.240740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773451.304733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773451.368737:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773451.368737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773451.368746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773451.432729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773451.496805:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773451.496805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773451.496842:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773451.624791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773451.624798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773451.624805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773451.624867:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773451.688770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773451.688795:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773451.688814:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773451.688814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773451.688856:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773451.688869:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773451.816721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773451.880707:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773451.944731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773452.008769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773452.008783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773452.008821:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773452.008835:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773452.072826:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773452.072837:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773452.072977:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773452.200731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773452.200783:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773452.264774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773452.328711:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773452.392735:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773452.392738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773452.392772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773452.456745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773452.520769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773452.520769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773452.520769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773452.648808:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773452.648831:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773452.648832:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773452.648832:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773452.712739:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773452.712764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773452.712788:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773452.712804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773452.712837:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773452.712840:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773452.840752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773452.904755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773452.968728:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773453.032822:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773453.032824:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773453.032825:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773453.032830:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773453.096791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773453.096840:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773453.096872:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773453.224795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773453.224796:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773453.288767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773453.352740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773453.416796:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773453.416796:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773453.416797:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773453.480718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773453.544753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773453.544753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773453.544753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773453.672761:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773453.672770:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773453.672770:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773453.672770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773453.736730:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773453.736732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773453.736748:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773453.736775:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773453.736803:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773453.736822:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773453.864715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773453.928802:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773453.992774:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790773453.992774:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773453.992781:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773453.992786:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773453.992795:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2449 00000100:00080000:1.0:1790773453.992806:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773453.992818:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773453.992855:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534679680/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773453.992882:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773453.992887:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773453.992889:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773453.992893:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2449 00000100:00080000:1.0:1790773453.992897:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773453.992903:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773453.992913:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773453.992930:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534679808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773453.992941:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773453.992945:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773453.992949:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773453.992953:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534679680:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773453.992959:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2449 00000100:00080000:1.0:1790773453.992969:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773453.992976:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773453.992982:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773453.992998:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534679936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773453.993008:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773453.993013:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773453.993015:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773453.993020:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2449 00000100:00080000:1.0:1790773453.993033:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773453.993041:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773453.993047:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773453.993085:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534680064/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773453.993100:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773453.993107:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773453.993110:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773453.993118:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534679808:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773453.993122:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2449 00000100:00080000:1.0:1790773453.993146:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773453.993153:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773453.993158:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773453.993171:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534680192/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773453.993186:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534679936:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773453.993189:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773453.993195:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773453.993198:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773453.993209:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534680064:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773453.993230:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2449 00000100:00080000:1.0:1790773453.993238:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773453.993245:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773453.993265:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773453.993284:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534680320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773453.993288:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534680192:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773453.993313:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773453.993319:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773453.993322:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773453.993328:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534680320:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773453.993333:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2449 00000100:00080000:1.0:1790773453.993341:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773453.993352:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773453.993360:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773453.993376:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534680448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773453.993407:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773453.993411:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773453.993413:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773453.993417:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2449 00000100:00080000:1.0:1790773453.993422:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773453.993429:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773453.993434:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773453.993443:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534680448:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773453.993451:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534680576/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773453.993467:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773453.993471:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773453.993474:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773453.993481:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534680576:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773453.993481:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2449 00000100:00080000:1.0:1790773453.993497:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773453.993510:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773453.993517:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773453.993532:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534680704/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773453.993583:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534680704:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773454.056754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773454.056764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773454.056764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773454.056764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773454.120732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773454.120805:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773454.120856:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773454.248732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773454.248739:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773454.312749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773454.376717:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773454.440723:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773454.440729:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773454.440729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773454.504781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773454.568761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773454.568761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773454.568761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773454.696800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773454.696801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773454.696815:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773454.696909:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773454.760752:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773454.760792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773454.760799:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773454.760801:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773454.760827:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773454.760843:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773454.888788:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773454.952741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773455.016873:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773455.080728:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773455.080761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773455.080813:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773455.080820:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773455.144712:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773455.144761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773455.144787:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773455.272702:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773455.272765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773455.336769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773455.400739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773455.464769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773455.464769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773455.464799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773455.528709:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773455.592756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773455.592756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773455.592756:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773455.720731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773455.720738:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773455.720747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773455.720790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773455.784808:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773455.784810:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773455.784811:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773455.784813:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773455.784849:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773455.784850:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773455.912734:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773455.976716:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773456.040873:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773456.104748:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773456.104748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773456.104748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773456.104753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773456.168734:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773456.168811:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773456.168824:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773456.296746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773456.296799:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773456.360749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773456.424769:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773456.488730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773456.488732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773456.488733:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773456.552722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773456.616839:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773456.616845:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773456.616845:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773456.744785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773456.744786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773456.744804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773456.744808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773456.808715:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773456.808748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773456.808759:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773456.808806:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773456.808807:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773456.808824:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773456.936757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773457.000723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773457.064873:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:1.0:1790773457.065053:0:2386:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:1.0:1790773457.065167:0:2386:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1790773457.065194:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773457.065297:0:2386:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:1.0:1790773457.065346:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773457.065347:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773453/real 1790773457] req@ffff9c4efac4e300 x1877759534679680/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790773457.065364:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773457.065380:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773457.065407:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773457.065426:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534679680/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790773457.065438:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:0.0:1790773457.065463:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000800:00000100:1.0:1790773457.065470:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:0.0:1790773457.065478:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773457.065488:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534679680:192.168.204.150@tcp:250:kworker.0 00000800:00000100:1.0:1790773457.065500:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773457.065534:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773453/real 1790773457] req@ffff9c4efac4df80 x1877759534680704/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773457.065555:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534680704/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773457.065569:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773457.065574:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773457.065579:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534680704:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773457.065587:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773453/real 1790773457] req@ffff9c4efac4dc00 x1877759534680576/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773457.065607:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534680576/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773457.065619:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773457.065624:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773457.065628:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534680576:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773457.065637:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773453/real 1790773457] req@ffff9c4efac4ed80 x1877759534680448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773457.065679:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534680448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773457.065689:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773457.065694:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773457.065698:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534680448:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773457.065707:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773453/real 1790773457] req@ffff9c4efac4c700 x1877759534680320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773457.065722:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534680320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773457.065742:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773457.065747:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773457.065751:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534680320:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773457.065759:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773453/real 1790773457] req@ffff9c4efac4f800 x1877759534680192/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773457.065773:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534680192/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773457.065783:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773457.065800:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773457.065805:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534680192:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773457.065813:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773453/real 1790773457] req@ffff9c4efac4ea00 x1877759534679808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773457.065831:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534679808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773457.065848:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773457.065853:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773457.065857:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534679808:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773457.065865:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773453/real 1790773457] req@ffff9c4efac4e680 x1877759534679936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773457.065879:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534679936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773457.065890:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773457.065900:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773457.065904:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534679936:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773457.065913:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773453/real 1790773457] req@ffff9c4efac4f480 x1877759534680064/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773457.065931:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534680064/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773457.065943:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773457.065947:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773457.065951:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534680064:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773457.128739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773457.128739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773457.128771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773457.128771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773457.192765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773457.192793:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773457.192897:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773457.320713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773457.320728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773457.384764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773457.448788:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773457.512794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773457.512814:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773457.512815:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773457.576822:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773457.640775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773457.640790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773457.640804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773457.768783:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773457.768792:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773457.768822:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773457.768822:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773457.832810:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773457.832813:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773457.832872:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773457.832876:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773457.832927:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773457.832928:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773457.960761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773458.024776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773458.088856:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773458.152783:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773458.152805:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773458.152806:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773458.152806:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773458.216746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773458.216755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773458.216846:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773458.344728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773458.344750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773458.408723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773458.472710:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773458.536720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773458.536729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773458.536756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773458.600734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773458.664748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773458.664748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773458.664753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773458.792793:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773458.792801:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773458.792806:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773458.792826:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773458.856830:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773458.856848:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773458.856859:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773458.856866:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773458.856911:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773458.856970:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773458.984761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773459.048772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773459.112844:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773459.112855:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773459.112862:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773459.112876:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2455 00000100:00100000:3.0:1790773459.112915:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773459.112933:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773459.112951:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773459.113015:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534680832/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773459.113065:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773459.113080:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773459.113093:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773459.113099:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2455 00000100:00080000:0.0:1790773459.113104:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773459.113111:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773459.113172:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773459.113193:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534680960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773459.113207:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773459.113212:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773459.113214:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773459.113238:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2455 00000100:00080000:0.0:1790773459.113243:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773459.113257:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773459.113259:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534680832:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773459.113266:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773459.113325:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534681088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773459.113345:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773459.113352:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773459.113365:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773459.113373:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2455 00000100:00080000:0.0:1790773459.113381:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773459.113393:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773459.113401:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773459.113410:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773459.113470:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773459/real 1790773459] req@ffff9c4ec7a7d180 x1877759534680832/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773459.113507:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534681216/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773459.113521:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534680832/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773459.113536:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773459.113550:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773459.113555:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773459.113556:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773459.113574:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773459.113576:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2455 00000100:00080000:0.0:1790773459.113589:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773459.113591:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534680832:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773459.113605:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773459.113616:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534680960:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773459.113619:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:1.0:1790773459.113723:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773459.113732:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534681344/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773459.113756:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773459.113756:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773459/real 1790773459] req@ffff9c4ec7a7c700 x1877759534680960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773459.113777:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773459.113785:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773459.113802:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2455 00000100:00080000:0.0:1790773459.113811:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773459.113813:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534680960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773459.113827:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773459.113847:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773459.113861:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773459.113875:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534681472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773459.113876:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773459.113901:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534680960:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773459.113906:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773459.113918:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773459.113923:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773459.113934:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2455 00000100:00080000:0.0:1790773459.113941:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773459.113956:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534681088:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773459.113957:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773459.113974:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773459.114006:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534681600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773459.114023:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773459.114032:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773459.114041:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773459.114045:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773459.114057:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2455 00000100:00100000:1.0:1790773459.114059:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773459/real 1790773459] req@ffff9c4ec7a7e300 x1877759534681088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773459.114066:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773459.114082:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773459.114098:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534681088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773459.114133:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773459.114142:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773459.114153:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773459.114165:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534681728/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773459.114166:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534681088:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773459.114196:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534681216:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773459.114199:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773459.114214:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773459.114218:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773459.114229:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2455 00000100:00080000:0.0:1790773459.114238:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773459.114259:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773459.114647:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773459.114698:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773459.114724:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534681856/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773459.114794:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773459/real 1790773459] req@ffff9c4ec7a7fb80 x1877759534681216/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773459.114821:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534681216/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773459.114841:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773459.114850:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773459.114859:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534681216:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773459.114873:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534681344:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773459.114920:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773459.114944:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773459/real 1790773459] req@ffff9c4ec7a7ca80 x1877759534681344/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773459.114967:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534681344/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773459.114984:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773459.114991:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773459.114998:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534681344:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773459.115009:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534681472:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773459.115049:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773459.115070:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773459/real 1790773459] req@ffff9c4ec7a7c380 x1877759534681472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773459.115091:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534681472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773459.115152:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773459.115160:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773459.115168:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534681472:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773459.115205:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534681600:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773459.115245:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773459.115268:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773459/real 1790773459] req@ffff9c4ec7a7d880 x1877759534681600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773459.115291:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534681600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773459.115307:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773459.115314:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773459.115322:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534681600:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773459.115334:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534681728:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773459.115372:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773459.115394:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773459/real 1790773459] req@ffff9c4ec7a7ed80 x1877759534681728/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773459.115413:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534681728/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773459.115431:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773459.115437:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773459.115442:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534681728:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773459.115452:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534681856:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773459.115484:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773459.115501:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773459/real 1790773459] req@ffff9c4ec7a7df80 x1877759534681856/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773459.115521:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534681856/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773459.115537:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773459.115542:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773459.115549:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534681856:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773459.176757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773459.176764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773459.176764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773459.176808:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773459.240765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773459.240816:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773459.240878:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773459.368723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773459.368726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773459.432760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773459.496745:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773459.560733:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773459.560754:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773459.560757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773459.624760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773459.688729:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773459.688731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773459.688753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773459.816757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773459.816758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773459.816781:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773459.816781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773459.880739:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773459.880754:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773459.880757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773459.880767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773459.880795:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773459.880812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773460.008748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773460.072721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773460.136868:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773460.200738:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773460.200775:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773460.200775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773460.200785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773460.264834:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773460.264856:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773460.264936:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773460.392774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773460.392810:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773460.456778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773460.520759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773460.584781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773460.584781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773460.584816:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773460.648758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773460.712713:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773460.712719:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773460.712723:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773460.840759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773460.840782:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773460.840785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773460.840785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773460.904773:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773460.904773:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773460.904787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773460.904787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773460.904818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773460.904819:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773461.032730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773461.096726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773461.160735:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773461.224752:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773461.224752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773461.224758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773461.224761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773461.288715:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773461.288715:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773461.288788:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773461.416722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773461.416722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773461.480724:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773461.544759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773461.608698:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773461.608718:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773461.608725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773461.672776:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773461.736776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773461.736782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773461.736792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773461.864777:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773461.864784:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773461.864810:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773461.864826:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773461.928784:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773461.928833:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773461.928835:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773461.928844:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773461.928872:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773461.928905:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773462.056820:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773462.120763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773462.184784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773462.248754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773462.248782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773462.248794:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773462.248794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773462.312773:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773462.312814:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773462.312973:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773462.440747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773462.440757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773462.504758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773462.568769:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773462.632766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773462.632782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773462.632790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773462.696777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773462.760771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773462.760786:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773462.760819:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773462.888753:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773462.888786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773462.888807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773462.888814:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773462.952784:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773462.952795:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773462.952813:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773462.952850:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773462.952872:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773462.952873:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773463.080782:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773463.144780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773463.208746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773463.272768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773463.272804:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773463.272814:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773463.272819:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773463.336775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773463.336831:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773463.336930:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773463.464751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773463.464764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773463.528737:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773463.592789:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773463.656753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773463.656784:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773463.656787:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773463.720762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773463.784753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773463.784775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773463.784775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773463.912779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773463.912779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773463.912785:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773463.912785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773463.976768:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773463.976817:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773463.976850:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773463.976851:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773463.976855:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773463.976910:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773464.104781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773464.168732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773464.232741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773464.232784:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773464.232791:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773464.232795:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773464.232805:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2460 00000100:00080000:0.0:1790773464.232825:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773464.232856:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773464.232906:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534681984/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773464.232943:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773464.232949:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773464.232952:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773464.232957:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2460 00000100:00080000:0.0:1790773464.232962:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773464.232977:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773464.232989:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773464.233039:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534682112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773464.233055:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773464.233061:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773464.233065:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773464.233090:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2460 00000100:00080000:0.0:1790773464.233098:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773464.233109:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773464.233120:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773464.233133:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534681984:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773464.233146:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534682240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773464.233168:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773464.233176:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773464.233180:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773464.233191:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2460 00000100:00080000:0.0:1790773464.233199:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773464.233225:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773464.233236:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773464.233278:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534682368/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773464.233279:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773464.233317:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773464.233330:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773464.233334:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773464.233340:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2460 00000100:00080000:0.0:1790773464.233345:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773464.233355:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773464.233383:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773464.233389:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773464/real 1790773464] req@ffff9c4ec7a7f480 x1877759534681984/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773464.233406:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534682496/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773464.233453:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534681984/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773464.233454:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773464.233493:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773464.233510:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773464.233519:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773464.233527:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773464.233534:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2460 00000100:00080000:0.0:1790773464.233541:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773464.233542:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534681984:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773464.233559:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773464.233566:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534682112:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773464.233569:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773464.233608:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534682624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773464.233621:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773464.233646:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773464.233691:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773464/real 1790773464] req@ffff9c4ec7a7df80 x1877759534682112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773464.233714:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534682112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773464.233723:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773464.233730:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773464.233768:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2460 00000100:00080000:0.0:1790773464.233782:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773464.233795:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773464.233801:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790773464.233803:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773464.233815:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773464.233835:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534682112:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773464.233852:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534682752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773464.233866:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773464.233871:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773464.233875:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773464.233882:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534682240:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773464.233883:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2460 00000100:00080000:0.0:1790773464.233901:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773464.233913:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773464.233921:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:1.0:1790773464.233936:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773464.233946:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534682880/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773464.233967:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773464/real 1790773464] req@ffff9c4ec7a7ed80 x1877759534682240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773464.233969:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773464.234003:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773464.234043:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773464.234077:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534682240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773464.234083:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2460 00000100:00080000:0.0:1790773464.234104:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773464.234107:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773464.234117:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773464.234123:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773464.234142:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534682240:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773464.234169:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534682368:192.168.204.150@tcp:8:kworker.0 00000008:00100000:0.0:1790773464.234170:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:1.0:1790773464.234216:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773464.234217:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534683008/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773464.234234:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773464/real 1790773464] req@ffff9c4ec7a7d880 x1877759534682368/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773464.234256:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534682368/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773464.234270:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773464.234276:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773464.234281:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534682368:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773464.234289:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534682496:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773464.234310:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773464.234332:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773464/real 1790773464] req@ffff9c4ec7a7c380 x1877759534682496/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773464.234347:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534682496/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773464.234368:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773464.234372:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773464.234377:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534682496:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773464.234390:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534682624:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773464.234410:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773464.234422:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773464/real 1790773464] req@ffff9c4ec7a7ca80 x1877759534682624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773464.234436:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534682624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773464.234456:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773464.234460:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773464.234465:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534682624:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773464.234473:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534682752:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773464.234494:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773464.234506:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773464/real 1790773464] req@ffff9c4ec7a7fb80 x1877759534682752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773464.234521:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534682752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773464.234532:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773464.234536:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773464.234541:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534682752:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773464.234565:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534682880:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773464.234586:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773464.234598:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773464/real 1790773464] req@ffff9c4ec7a7e300 x1877759534682880/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773464.234612:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534682880/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773464.234625:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773464.234630:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773464.234634:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534682880:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773464.234642:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534683008:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773464.234697:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773464.234711:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773464/real 1790773464] req@ffff9c4ec7a7c700 x1877759534683008/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773464.234735:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534683008/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773464.234748:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773464.234753:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773464.234758:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534683008:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773464.296747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773464.296747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773464.296769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773464.296770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773464.360760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773464.360803:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773464.360873:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773464.488752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773464.488764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773464.552780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773464.616751:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773464.680763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773464.680784:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773464.680785:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773464.744844:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773464.808820:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773464.808820:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773464.808850:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773464.936789:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773464.936800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773464.936840:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773464.936851:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773465.000768:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773465.000769:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773465.000780:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773465.000833:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773465.000834:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773465.000847:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773465.128746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773465.192743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773465.256768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773465.320722:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773465.320783:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773465.320820:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773465.320820:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773465.384836:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773465.384876:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773465.384937:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773465.512774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773465.512818:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773465.576814:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773465.640789:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773465.704819:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773465.704838:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773465.704865:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773465.768777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773465.832727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773465.832770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773465.832796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773465.960724:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773465.960731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773465.960815:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773465.960823:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773466.024746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773466.024754:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773466.024773:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773466.024773:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773466.024803:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773466.024804:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773466.152763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773466.216721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773466.280720:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773466.344768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773466.344769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773466.344778:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773466.344783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773466.408733:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773466.408770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773466.408831:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773466.536717:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773466.536727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773466.600722:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773466.664744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773466.728759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773466.728768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773466.728769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773466.792755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773466.856782:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773466.856782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773466.856782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773466.984721:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773466.984731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773466.984732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773466.984741:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773467.048750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773467.048770:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773467.048779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773467.048799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773467.048829:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773467.048835:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773467.176712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773467.240756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773467.304835:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773467.368766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773467.368766:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773467.368801:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773467.368810:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773467.432800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773467.432826:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773467.432869:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773467.560761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773467.560773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773467.624781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773467.688862:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773467.752748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773467.752748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773467.752775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773467.816774:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773467.880842:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773467.880851:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773467.880852:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773468.008797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773468.008807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773468.008811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773468.008811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773468.072760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773468.072798:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773468.072817:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773468.072818:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773468.072862:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773468.072903:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773468.200764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773468.264772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773468.328718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773468.392729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773468.392730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773468.392736:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773468.392790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773468.456733:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773468.456766:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773468.456812:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773468.584755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773468.584755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773468.648809:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773468.712797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773468.776773:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773468.776773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773468.776780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773468.840765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773468.904773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773468.904773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773468.904773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773469.032742:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773469.032744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773469.032780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773469.032821:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773469.096749:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773469.096761:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773469.096768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773469.096768:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773469.096798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773469.096810:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773469.224748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773469.288749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773469.352797:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773469.352807:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773469.352817:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773469.352824:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773469.352838:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2465 00000100:00080000:1.0:1790773469.352852:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773469.352868:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773469.352945:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534683136/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773469.353001:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773469.353009:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773469.353014:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773469.353023:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2465 00000100:00080000:1.0:1790773469.353031:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773469.353040:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773469.353054:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773469.353080:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534683264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773469.353100:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773469.353102:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534683136:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773469.353110:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773469.353117:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773469.353126:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2465 00000100:00080000:1.0:1790773469.353131:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773469.353137:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773469.353146:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773469.353165:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534683392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773469.353180:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773469.353186:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773469.353188:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773469.353194:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2465 00000100:00080000:1.0:1790773469.353199:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773469.353205:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773469.353212:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:0.0:1790773469.353221:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773469.353233:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534683520/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773469.353264:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773469.353274:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773469.353277:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773469.353284:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2465 00000100:00080000:1.0:1790773469.353290:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773469.353301:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773469.353304:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773469/real 1790773469] req@ffff9c4efac4fb80 x1877759534683136/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773469.353309:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773469.353333:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534683648/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773469.353348:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773469.353354:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534683136/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773469.353356:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773469.353362:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773469.353376:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2465 00000100:00080000:1.0:1790773469.353384:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773469.353387:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773469.353394:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773469.353404:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773469.353417:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534683776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773469.353419:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773469.353436:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534683136:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773469.353442:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773469.353459:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773469.353464:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773469.353471:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2465 00000100:00080000:1.0:1790773469.353477:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773469.353482:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773469.353489:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773469.353492:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534683264:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773469.353507:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534683904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773469.353525:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773469.353531:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773469.353534:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773469.353543:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773469.353544:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2465 00000100:00080000:1.0:1790773469.353557:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773469.353565:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773469.353574:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773469.353581:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773469/real 1790773469] req@ffff9c4efac4f480 x1877759534683264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773469.353597:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534684032/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773469.353614:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534683264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773469.353616:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773469.353630:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773469.353636:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773469.353643:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2465 00000100:00080000:1.0:1790773469.353685:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773469.353693:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773469.353699:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773469.353725:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534684160/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773469.353736:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773469.353752:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773469.353763:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534683264:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773469.353780:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534683392:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773469.353828:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773469.353855:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773469/real 1790773469] req@ffff9c4efac4e680 x1877759534683392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773469.353883:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534683392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773469.353902:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773469.353909:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773469.353946:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534683392:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773469.353961:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534683520:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773469.354008:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773469.354028:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773469/real 1790773469] req@ffff9c4efac4ea00 x1877759534683520/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773469.354046:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534683520/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773469.354060:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773469.354065:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773469.354069:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534683520:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773469.354078:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534683648:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773469.354102:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773469.354116:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773469/real 1790773469] req@ffff9c4efac4f800 x1877759534683648/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773469.354140:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534683648/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773469.354159:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773469.354167:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773469.354174:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534683648:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773469.354186:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534683776:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773469.354229:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773469.354254:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773469/real 1790773469] req@ffff9c4efac4c700 x1877759534683776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773469.354278:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534683776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773469.354323:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773469.354331:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773469.354336:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534683776:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773469.354364:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534683904:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773469.354394:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773469.354411:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773469/real 1790773469] req@ffff9c4efac4ed80 x1877759534683904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773469.354426:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534683904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773469.354437:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773469.354441:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773469.354445:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534683904:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773469.354452:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534684032:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773469.354472:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773469.354484:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773469/real 1790773469] req@ffff9c4efac4dc00 x1877759534684032/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773469.354499:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534684032/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773469.354512:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773469.354516:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773469.354521:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534684032:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773469.354528:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534684160:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773469.354548:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773469.354559:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773469/real 1790773469] req@ffff9c4efac4df80 x1877759534684160/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773469.354573:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534684160/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773469.354585:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773469.354589:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773469.354594:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534684160:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773469.416789:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773469.416800:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773469.416801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773469.416813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773469.480782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773469.480895:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773469.480901:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773469.608759:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773469.608784:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773469.672802:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773469.736766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773469.800779:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773469.800783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773469.800783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773469.864832:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773469.928727:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773469.928793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773469.928800:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773470.056765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773470.056774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773470.056775:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773470.056780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773470.120749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773470.120756:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773470.120760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773470.120800:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773470.120819:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773470.120820:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773470.248717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773470.312750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773470.376703:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773470.440759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773470.440769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773470.440771:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773470.440771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773470.504720:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773470.504782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773470.504790:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773470.632704:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773470.632756:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773470.696765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773470.760732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773470.824801:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773470.824801:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773470.824807:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773470.888739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773470.952752:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773470.952753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773470.952769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773471.080738:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773471.080747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773471.080747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773471.080843:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773471.144741:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773471.144749:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773471.144798:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773471.144824:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773471.144831:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773471.144872:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773471.272708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773471.336703:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773471.400822:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773471.464746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773471.464746:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773471.464746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773471.464747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773471.528733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773471.528739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773471.528785:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773471.656722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773471.656722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773471.720717:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773471.784730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773471.848727:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773471.848734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773471.848752:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773471.912714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773471.976737:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773471.976738:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773471.976751:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773472.104756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773472.104756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773472.104756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773472.104756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773472.168737:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773472.168746:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773472.168760:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773472.168765:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773472.168785:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773472.168786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773472.296740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773472.360718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773472.424706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773472.488725:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773472.488725:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773472.488725:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773472.488731:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773472.552728:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773472.552749:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773472.552819:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773472.680721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773472.680725:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773472.744738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773472.808739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773472.872712:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773472.872716:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773472.872773:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773472.936748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773473.000739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773473.000757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773473.000760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773473.128739:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773473.128740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773473.128740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773473.128740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773473.192730:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773473.192755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773473.192755:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773473.192774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773473.192795:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773473.192804:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773473.320729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773473.384783:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773473.448717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773473.512770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773473.512770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773473.512770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773473.512780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773473.576758:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773473.576817:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773473.576921:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773473.704745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773473.704746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773473.768775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773473.832759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773473.896717:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773473.896717:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773473.896725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773473.960713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773474.024743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773474.024747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773474.024756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773474.152764:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773474.152764:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773474.152765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773474.152765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773474.216742:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773474.216749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773474.216749:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773474.216773:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773474.216808:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773474.216813:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773474.344710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773474.408788:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773474.472720:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773474.472740:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773474.472744:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1790773474.472749:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773474.472753:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2470 00000100:00080000:1.0:1790773474.472763:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773474.472773:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773474.472831:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534684288/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773474.472860:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773474.472879:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773474.472881:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773474.472886:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2470 00000100:00080000:1.0:1790773474.472890:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773474.472896:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773474.472908:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773474.472924:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534684288:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773474.472927:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534684416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773474.472943:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773474.472946:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773474.472948:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773474.472951:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2470 00000100:00080000:1.0:1790773474.472955:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773474.472960:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773474.472965:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773474.472975:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534684544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773474.472983:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773474.472986:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773474.472988:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773474.472992:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773474.472995:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2470 00000100:00080000:1.0:1790773474.472999:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773474.473002:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773474.473007:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773474.473016:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534684672/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773474.473023:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773474.473026:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773474.473028:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773474.473032:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2470 00000100:00080000:1.0:1790773474.473034:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773474.473039:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773474/real 1790773474] req@ffff9c4efac4e300 x1877759534684288/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773474.473039:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773474.473056:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773474.473064:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534684800/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773474.473075:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773474.473082:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773474.473085:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773474.473091:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2470 00000100:00080000:1.0:1790773474.473095:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773474.473099:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773474.473103:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773474.473109:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534684288/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773474.473111:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534684928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773474.473142:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773474.473145:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773474.473146:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773474.473149:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773474.473153:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773474.473156:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2470 00000100:00100000:0.0:1790773474.473161:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534684288:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773474.473161:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773474.473167:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773474.473171:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773474.473178:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534685056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773474.473186:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773474.473187:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534684416:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773474.473190:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773474.473194:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773474.473198:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2470 00000100:00080000:1.0:1790773474.473200:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773474.473205:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773474.473219:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:0.0:1790773474.473226:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773474.473230:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534685184/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773474.473239:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773474.473241:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773474/real 1790773474] req@ffff9c4efac4df80 x1877759534684416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773474.473243:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773474.473247:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773474.473255:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2470 00000100:00080000:1.0:1790773474.473259:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773474.473263:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534684416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773474.473264:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773474.473270:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773474.473293:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534685312/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773474.473299:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773474.473311:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773474.473317:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534684416:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773474.473326:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534684544:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773474.473359:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773474.473372:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773474/real 1790773474] req@ffff9c4efac4dc00 x1877759534684544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773474.473380:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534684544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773474.473391:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773474.473394:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773474.473396:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534684544:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773474.473400:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534684672:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773474.473426:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773474.473432:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773474/real 1790773474] req@ffff9c4efac4ed80 x1877759534684672/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773474.473440:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534684672/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773474.473447:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773474.473449:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773474.473451:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534684672:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773474.473455:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534684800:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773474.473465:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773474.473471:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773474/real 1790773474] req@ffff9c4efac4c700 x1877759534684800/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773474.473490:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534684800/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773474.473496:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773474.473498:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773474.473500:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534684800:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773474.473504:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534684928:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773474.473514:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773474.473520:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773474/real 1790773474] req@ffff9c4efac4f800 x1877759534684928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773474.473528:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534684928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773474.473550:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773474.473552:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773474.473554:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534684928:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773474.473567:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534685056:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773474.473578:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773474.473585:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773474/real 1790773474] req@ffff9c4efac4ea00 x1877759534685056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773474.473593:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534685056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773474.473598:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773474.473600:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773474.473602:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534685056:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773474.473617:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534685184:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773474.473627:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773474.473633:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773474/real 1790773474] req@ffff9c4efac4e680 x1877759534685184/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773474.473641:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534685184/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773474.473648:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773474.473673:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773474.473676:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534685184:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773474.473680:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534685312:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773474.473693:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773474.473700:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773474/real 1790773474] req@ffff9c4efac4f480 x1877759534685312/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773474.473708:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534685312/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773474.473715:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773474.473717:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773474.473719:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534685312:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773474.536766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773474.536766:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773474.536767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773474.536774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773474.600744:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773474.600745:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773474.600783:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773474.728728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773474.728746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773474.792807:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773474.856733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773474.920697:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773474.920713:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773474.920733:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773474.984711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773475.048731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773475.048731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773475.048734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773475.176755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773475.176758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773475.176759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773475.176761:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773475.240762:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773475.240776:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773475.240777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773475.240791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773475.240817:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773475.240830:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773475.368737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773475.432791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773475.496708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773475.560771:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773475.560771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773475.560771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773475.560772:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773475.624724:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773475.624747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773475.624832:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773475.752718:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773475.752741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773475.816737:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773475.880711:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773475.944708:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773475.944725:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773475.944726:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773476.008740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773476.072731:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773476.072732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773476.072736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773476.200741:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773476.200755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773476.200756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773476.200756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773476.264709:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773476.264729:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773476.264734:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773476.264753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773476.264758:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773476.264770:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773476.392725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773476.456835:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773476.520756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773476.584749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773476.584768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773476.584824:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773476.584872:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773476.648892:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773476.648964:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773476.649059:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773476.776739:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773476.776762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773476.840752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773476.904846:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773476.968753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773476.968797:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773476.968812:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773477.032786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773477.096821:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773477.096822:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773477.096844:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773477.224758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773477.224803:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773477.224807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773477.224815:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773477.288817:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773477.288817:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773477.288833:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773477.288867:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773477.288884:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773477.288884:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773477.416782:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773477.480791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773477.544794:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773477.608780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773477.608794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773477.608813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773477.608826:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773477.672775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773477.672825:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773477.672963:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773477.800802:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773477.800862:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773477.864803:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773477.928851:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773477.992811:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773477.992821:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773477.992913:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773478.056803:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773478.120732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773478.120779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773478.120790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773478.248740:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773478.248740:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773478.248740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773478.248883:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773478.312757:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773478.312757:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773478.312793:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773478.312800:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773478.312803:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773478.312805:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773478.440722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773478.504728:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773478.568727:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773478.632858:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773478.632858:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773478.632866:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773478.632868:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773478.696790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773478.696798:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773478.696848:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773478.824779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773478.824779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773478.888742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773478.952779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773479.016741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773479.016745:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773479.016837:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773479.080765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773479.144808:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773479.144809:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773479.144823:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773479.272831:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773479.272879:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773479.272912:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773479.272913:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773479.336820:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773479.336888:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773479.336894:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773479.336897:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773479.336913:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773479.336965:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773479.464794:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773479.528789:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773479.592814:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773479.592832:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773479.592842:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773479.592847:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773479.592859:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2475 00000100:00080000:1.0:1790773479.592872:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773479.592886:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773479.592962:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534685440/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773479.593022:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773479.593031:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773479.593035:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773479.593044:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2475 00000100:00080000:1.0:1790773479.593052:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773479.593062:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773479.593082:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773479.593111:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534685568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773479.593123:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534685440:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773479.593133:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773479.593139:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773479.593143:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773479.593152:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2475 00000100:00080000:1.0:1790773479.593158:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773479.593166:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773479.593175:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773479.593193:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534685696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773479.593210:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773479.593217:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773479.593221:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773479.593229:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2475 00000800:00000100:0.0:1790773479.593233:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773479.593235:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773479.593244:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773479.593254:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773479.593284:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534685824/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773479.593294:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773479.593297:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773479.593299:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773479.593304:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2475 00000100:00080000:1.0:1790773479.593307:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773479.593310:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773479/real 1790773479] req@ffff9c4efac4fb80 x1877759534685440/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773479.593312:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773479.593317:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773479.593328:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534685952/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773479.593331:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534685440/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773479.593340:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773479.593343:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773479.593345:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773479.593345:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773479.593350:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2475 00000100:00080000:0.0:1790773479.593351:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773479.593354:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773479.593356:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534685440:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773479.593359:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773479.593364:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773479.593373:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534686080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773479.593382:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534685568:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773479.593382:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773479.593387:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773479.593388:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773479.593393:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2475 00000100:00080000:1.0:1790773479.593396:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773479.593404:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773479.593405:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773479.593409:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773479.593420:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534686208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773479.593421:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773479/real 1790773479] req@ffff9c4efac4f480 x1877759534685568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773479.593433:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773479.593434:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534685568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773479.593437:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773479.593439:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773479.593444:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2475 00000100:00080000:1.0:1790773479.593448:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773479.593453:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773479.593455:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773479.593457:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773479.593459:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773479.593463:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534685568:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773479.593467:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534686336/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773479.593470:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534685696:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773479.593478:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773479.593482:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773479.593484:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773479.593488:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2475 00000800:00000100:0.0:1790773479.593489:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773479.593492:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773479.593498:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773479.593500:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773479/real 1790773479] req@ffff9c4efac4e680 x1877759534685696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773479.593502:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773479.593513:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534686464/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773479.593515:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534685696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773479.593526:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773479.593530:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773479.593534:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534685696:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773479.593541:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534685824:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773479.593564:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773479.593576:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773479/real 1790773479] req@ffff9c4efac4ea00 x1877759534685824/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773479.593604:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534685824/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773479.593626:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773479.593634:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773479.593641:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534685824:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773479.593654:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534685952:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773479.593769:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773479.593779:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773479/real 1790773479] req@ffff9c4efac4f800 x1877759534685952/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773479.593788:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534685952/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773479.593801:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773479.593804:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773479.593806:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534685952:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773479.593847:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534686080:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773479.593875:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773479.593889:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773479/real 1790773479] req@ffff9c4efac4c700 x1877759534686080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773479.593905:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534686080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773479.593925:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773479.593930:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773479.593934:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534686080:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773479.593942:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534686208:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773479.593961:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773479.593990:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773479/real 1790773479] req@ffff9c4efac4ed80 x1877759534686208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773479.594003:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534686208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773479.594011:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773479.594014:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773479.594019:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534686208:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773479.594025:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534686336:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773479.594043:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773479.594052:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773479/real 1790773479] req@ffff9c4efac4dc00 x1877759534686336/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773479.594064:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534686336/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773479.594074:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773479.594078:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773479.594081:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534686336:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773479.594087:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534686464:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773479.594102:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773479.594111:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773479/real 1790773479] req@ffff9c4efac4df80 x1877759534686464/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773479.594123:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534686464/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773479.594133:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773479.594136:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773479.594140:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534686464:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773479.656826:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773479.656826:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773479.656833:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773479.656855:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773479.720770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773479.720884:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773479.720934:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773479.848721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773479.848733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773479.912749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773479.976737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773480.040766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773480.040775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773480.040782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773480.104793:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773480.168791:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773480.168793:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773480.168801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773480.296806:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773480.296813:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773480.296814:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773480.296846:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773480.360812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773480.360814:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773480.360830:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773480.360874:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773480.360915:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773480.360916:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773480.488796:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773480.552826:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773480.616721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773480.680793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773480.680801:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773480.680833:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773480.680886:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773480.744799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773480.744837:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773480.744989:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773480.872735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773480.872757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773480.936816:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773481.000797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773481.064884:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773481.064888:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773481.064889:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773481.128786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773481.192770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773481.192782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773481.192811:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773481.320807:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773481.320858:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773481.320861:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773481.320877:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773481.384802:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773481.384804:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773481.384830:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773481.384852:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773481.384895:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773481.384908:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773481.512839:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773481.576783:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773481.640809:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773481.704789:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773481.704799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773481.704804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773481.704857:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773481.768782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773481.768816:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773481.768853:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773481.896769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773481.896800:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773481.960785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773482.024797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773482.088751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773482.088798:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773482.088798:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773482.152760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773482.216755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773482.216783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773482.216783:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773482.344824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773482.344828:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773482.344840:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773482.344868:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773482.408819:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773482.408822:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773482.408829:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773482.408857:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773482.408897:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773482.408897:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773482.536756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773482.600762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773482.664794:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773482.728750:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773482.728768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773482.728790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773482.728791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773482.792746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773482.792794:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773482.792823:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773482.920760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773482.920774:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773482.984742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773483.048761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773483.112725:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773483.112744:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773483.112748:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773483.176797:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773483.240736:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773483.240739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773483.240746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773483.368770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773483.368780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773483.368780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773483.368823:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773483.432787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773483.432823:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773483.432828:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773483.432839:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773483.432860:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773483.432872:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773483.560742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773483.624781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773483.688752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773483.752760:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773483.752761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773483.752771:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773483.752771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773483.816756:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773483.816791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773483.816849:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773483.944734:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773483.944766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773484.008801:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773484.072740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773484.136774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773484.136774:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773484.136779:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773484.200757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773484.264794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773484.264794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773484.264795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773484.392744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773484.392752:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773484.392755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773484.392757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773484.456762:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773484.456769:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773484.456779:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773484.456801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773484.456821:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773484.456837:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773484.584747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773484.648752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773484.712781:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790773484.712781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773484.712795:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773484.712801:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773484.712812:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2480 00000100:00080000:1.0:1790773484.712843:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773484.712855:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773484.712894:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534686592/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773484.712937:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773484.712945:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773484.712947:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773484.712951:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2480 00000100:00080000:1.0:1790773484.712956:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773484.712960:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773484.712979:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773484.712993:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534686720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773484.713005:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773484.713009:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773484.713011:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773484.713016:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2480 00000100:00080000:1.0:1790773484.713019:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773484.713026:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534686592:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773484.713027:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773484.713038:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773484.713051:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534686848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773484.713076:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773484.713082:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773484.713084:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773484.713088:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2480 00000100:00080000:1.0:1790773484.713100:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773484.713105:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773484.713141:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:0.0:1790773484.713148:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773484.713163:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534686976/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773484.713177:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773484.713181:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773484.713184:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773484.713188:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2480 00000100:00080000:1.0:1790773484.713192:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773484.713198:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773484.713205:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773484.713219:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773484/real 1790773484] req@ffff9c4efac4e300 x1877759534686592/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773484.713221:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534687104/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773484.713257:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773484.713266:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773484.713268:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773484.713273:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2480 00000100:00080000:1.0:1790773484.713277:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773484.713287:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534686592/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773484.713287:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773484.713297:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773484.713315:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790773484.713315:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534687232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773484.713331:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773484.713337:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773484.713342:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534686592:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773484.713345:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773484.713354:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773484.713361:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534686720:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773484.713370:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2480 00000100:00080000:1.0:1790773484.713379:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773484.713390:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773484.713402:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773484.713412:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773484.713427:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773484/real 1790773484] req@ffff9c4efac4df80 x1877759534686720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773484.713442:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534687360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773484.713447:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534686720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773484.713491:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773484.713496:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773484.713499:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773484.713502:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773484.713509:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2480 00000100:00080000:0.0:1790773484.713516:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773484.713519:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773484.713524:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534686720:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773484.713550:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773484.713560:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773484.713583:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534686848:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773484.713584:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534687488/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773484.713611:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773484.713666:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773484.713669:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773484.713710:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2480 00000100:00080000:1.0:1790773484.713735:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773484.713736:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773484.713759:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773484.713764:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773484/real 1790773484] req@ffff9c4efac4dc00 x1877759534686848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773484.713773:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773484.713785:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534686848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773484.713803:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534687616/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773484.713806:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773484.713839:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773484.713850:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534686848:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773484.713863:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534686976:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773484.713904:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773484.713917:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773484/real 1790773484] req@ffff9c4efac4ed80 x1877759534686976/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773484.713928:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534686976/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773484.713938:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773484.713941:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773484.713945:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534686976:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773484.713951:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534687104:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773484.713977:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773484.713986:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773484/real 1790773484] req@ffff9c4efac4c700 x1877759534687104/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773484.713997:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534687104/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773484.714005:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773484.714008:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773484.714011:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534687104:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773484.714016:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534687232:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773484.714029:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773484.714038:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773484/real 1790773484] req@ffff9c4efac4f800 x1877759534687232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773484.714048:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534687232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773484.714070:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773484.714075:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773484.714098:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534687232:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773484.714109:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534687360:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773484.714153:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773484.714176:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773484/real 1790773484] req@ffff9c4efac4ea00 x1877759534687360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773484.714204:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534687360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773484.714221:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773484.714228:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773484.714235:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534687360:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773484.714271:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534687488:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773484.714310:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773484.714327:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773484/real 1790773484] req@ffff9c4efac4e680 x1877759534687488/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773484.714342:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534687488/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773484.714356:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773484.714360:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773484.714365:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534687488:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773484.714373:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534687616:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773484.714393:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773484.714404:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773484/real 1790773484] req@ffff9c4efac4f480 x1877759534687616/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773484.714417:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534687616/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773484.714447:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773484.714452:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773484.714456:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534687616:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773484.776776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773484.776779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773484.776786:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773484.776820:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773484.840713:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773484.840763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773484.840777:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773484.968742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773484.968749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773485.032828:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773485.096744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773485.160769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773485.160769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773485.160778:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773485.224768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773485.288784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773485.288832:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773485.288839:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773485.416768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773485.416776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773485.416778:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773485.416805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773485.480754:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773485.480768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773485.480802:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773485.480804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773485.480825:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773485.480865:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773485.608709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773485.672730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773485.736738:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773485.800746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773485.800746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773485.800746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773485.800747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773485.864745:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773485.864765:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773485.864791:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773485.992722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773485.992722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773486.056781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773486.120755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773486.184793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773486.184794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773486.184797:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773486.248752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773486.312792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773486.312792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773486.312795:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773486.440729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773486.440743:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773486.440743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773486.440745:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773486.504735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773486.504735:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773486.504737:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773486.504742:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773486.504772:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773486.504775:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773486.632751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773486.696780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773486.760716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773486.824734:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773486.824734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773486.824734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773486.824738:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773486.888704:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773486.888723:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773486.888792:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773487.016723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773487.016732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773487.080722:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773487.144711:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773487.208719:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773487.208719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773487.208724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773487.272717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773487.336761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773487.336761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773487.336788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773487.464755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773487.464755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773487.464755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773487.464756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773487.528727:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773487.528727:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773487.528746:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773487.528752:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773487.528774:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773487.528777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773487.656726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773487.720728:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773487.784705:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773487.848756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773487.848756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773487.848761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773487.848762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773487.912743:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773487.912763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773487.912807:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773488.040732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773488.040732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773488.104769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773488.168747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773488.232753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773488.232753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773488.232754:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773488.296727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773488.360735:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773488.360739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773488.360742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773488.488720:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773488.488720:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773488.488730:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773488.488863:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773488.552777:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773488.552777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773488.552777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773488.552798:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773488.552831:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773488.552838:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773488.680729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773488.744752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773488.808715:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773488.872742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773488.872742:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773488.872749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773488.872779:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773488.936793:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773488.936824:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773488.936865:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773489.064754:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773489.064816:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773489.128733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773489.192772:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773489.256723:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773489.256745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773489.256768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773489.320770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773489.384773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773489.384789:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773489.384845:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773489.512794:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773489.512840:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773489.512840:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773489.512845:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773489.576762:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773489.576805:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773489.576808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773489.576824:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773489.576854:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773489.576861:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773489.704766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773489.768763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773489.832767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773489.832843:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773489.832855:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773489.832862:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773489.832877:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2485 00000100:00080000:0.0:1790773489.832892:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773489.832909:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773489.832960:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534687744/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773489.833001:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773489.833005:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773489.833007:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773489.833011:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2485 00000100:00080000:0.0:1790773489.833015:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773489.833020:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773489.833031:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773489.833046:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534687872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773489.833059:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773489.833063:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773489.833066:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773489.833070:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2485 00000100:00080000:0.0:1790773489.833072:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773489.833079:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773489.833084:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773489.833102:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534688000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773489.833112:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534687744:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773489.833118:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773489.833137:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773489.833141:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773489.833147:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2485 00000100:00080000:0.0:1790773489.833150:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773489.833157:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773489.833166:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773489.833186:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534688128/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773489.833203:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773489.833209:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773489.833212:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773489.833219:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2485 00000100:00080000:0.0:1790773489.833228:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773489.833236:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773489.833249:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:1.0:1790773489.833249:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773489.833274:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534688256/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773489.833286:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773489.833290:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773489.833293:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773489.833299:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2485 00000100:00080000:0.0:1790773489.833303:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773489.833309:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773489.833315:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773489.833328:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534688384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773489.833350:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773489.833351:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773489/real 1790773489] req@ffff9c4ec7a7d180 x1877759534687744/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773489.833363:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773489.833373:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773489.833393:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2485 00000100:00080000:0.0:1790773489.833417:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773489.833428:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773489.833435:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773489.833439:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534687744/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773489.833455:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534688512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773489.833465:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773489.833468:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773489.833476:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773489.833477:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773489.833484:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773489.833497:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2485 00000100:00100000:1.0:1790773489.833498:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534687744:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773489.833507:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773489.833562:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773489.833569:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534687872:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773489.833573:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773489.833617:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534688640/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773489.833635:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773489.833639:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773489.833642:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773489.833649:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2485 00000800:00000100:1.0:1790773489.833708:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773489.833714:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773489.833730:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773489.833741:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773489.833754:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773489/real 1790773489] req@ffff9c4ec7a7c700 x1877759534687872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773489.833763:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534688768/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773489.833862:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534687872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773489.833898:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773489.833904:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773489.833910:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534687872:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773489.833951:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534688000:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773489.833992:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773489.834018:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773489/real 1790773489] req@ffff9c4ec7a7e300 x1877759534688000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773489.834045:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534688000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773489.834065:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773489.834073:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773489.834081:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534688000:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773489.834093:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534688128:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773489.834127:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773489.834143:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773489/real 1790773489] req@ffff9c4ec7a7fb80 x1877759534688128/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773489.834159:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534688128/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773489.834173:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773489.834179:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773489.834183:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534688128:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773489.834192:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534688256:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773489.834212:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773489.834224:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773489/real 1790773489] req@ffff9c4ec7a7ca80 x1877759534688256/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773489.834242:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534688256/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773489.834262:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773489.834269:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773489.834276:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534688256:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773489.834289:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534688384:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773489.834330:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773489.834354:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773489/real 1790773489] req@ffff9c4ec7a7c380 x1877759534688384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773489.834372:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534688384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773489.834393:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773489.834409:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773489.834414:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534688384:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773489.834423:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534688512:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773489.834452:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773489.834475:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773489/real 1790773489] req@ffff9c4ec7a7d880 x1877759534688512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773489.834490:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534688512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773489.834511:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773489.834546:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773489.834554:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534688512:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773489.834569:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534688640:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773489.834612:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773489.834637:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773489/real 1790773489] req@ffff9c4ec7a7ed80 x1877759534688640/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773489.834701:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534688640/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773489.834716:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773489.834721:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773489.834728:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534688640:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773489.834742:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534688768:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773489.834785:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773489.834811:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773489/real 1790773489] req@ffff9c4ec7a7df80 x1877759534688768/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773489.834847:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534688768/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773489.834861:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773489.834866:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773489.834872:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534688768:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773489.896764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773489.896807:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773489.896808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773489.896809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773489.960793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773489.960817:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773489.960909:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773490.088769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773490.088774:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773490.152771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773490.216794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773490.280774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773490.280811:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773490.280832:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773490.344733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773490.408811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773490.408812:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773490.408883:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773490.536761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773490.536761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773490.536761:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773490.536829:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773490.600755:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773490.600764:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773490.600806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773490.600845:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773490.600883:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773490.600895:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773490.728776:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773490.792713:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773490.856722:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773490.920820:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773490.920828:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773490.920830:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773490.920831:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773490.984740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773490.984766:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773490.984874:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773491.112732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773491.112739:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773491.176811:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773491.240754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773491.304708:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773491.304737:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773491.304799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773491.368742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773491.432746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773491.432770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773491.432794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773491.560749:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773491.560749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773491.560791:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773491.560792:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773491.624761:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773491.624770:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773491.624806:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773491.624807:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773491.624888:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773491.624889:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773491.752758:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773491.816856:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773491.880758:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773491.944785:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773491.944802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773491.944809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773491.944810:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773492.008769:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773492.008799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773492.008854:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773492.136721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773492.136735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773492.200783:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773492.264756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773492.328786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773492.328786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773492.328797:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773492.392777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773492.456775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773492.456776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773492.456792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773492.584743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773492.584754:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773492.584756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773492.584815:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773492.648719:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773492.648723:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773492.648724:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773492.648745:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773492.648759:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773492.648778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773492.776706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773492.840769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773492.904723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773492.968770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773492.968770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773492.968771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773492.968779:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773493.032756:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773493.032774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773493.032905:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773493.160710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773493.160742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773493.224753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773493.288773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773493.352702:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773493.352717:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773493.352749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773493.416736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773493.480725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773493.480748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773493.480757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773493.608708:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773493.608721:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773493.608724:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773493.608787:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773493.672739:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773493.672770:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773493.672792:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773493.672792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773493.672794:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773493.672822:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773493.800722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773493.864764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773493.928744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773493.992843:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773493.992846:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773493.992862:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773493.992880:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773494.056801:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773494.056813:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773494.056912:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773494.184744:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773494.184744:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773494.248741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773494.312744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773494.376712:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773494.376712:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773494.376740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773494.440767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773494.504727:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773494.504727:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773494.504728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773494.632754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773494.632758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773494.632759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773494.632779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773494.696751:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773494.696751:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773494.696780:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773494.696801:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773494.696806:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773494.696808:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773494.824740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773494.888734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773494.952726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773494.952760:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773494.952768:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773494.952773:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773494.952781:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2490 00000100:00080000:0.0:1790773494.952790:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773494.952800:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773494.952835:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534688896/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773494.952859:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773494.952862:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773494.952863:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773494.952867:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2490 00000100:00080000:0.0:1790773494.952869:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773494.952874:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773494.952882:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773494.952891:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534689024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773494.952898:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773494.952901:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773494.952903:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773494.952905:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2490 00000100:00080000:0.0:1790773494.952907:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773494.952911:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773494.952915:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773494.952924:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534689152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773494.952932:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773494.952935:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773494.952936:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773494.952940:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2490 00000100:00080000:0.0:1790773494.952942:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773494.952951:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773494.952952:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534688896:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773494.952955:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773494.952974:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534689280/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773494.952983:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773494.952986:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773494.952990:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773494.952995:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2490 00000100:00080000:0.0:1790773494.953000:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773494.953006:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773494.953012:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773494.953027:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534689408/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773494.953040:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773494.953045:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773494.953048:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773494.953056:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2490 00000800:00000100:1.0:1790773494.953060:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773494.953068:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773494.953074:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773494.953080:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773494.953094:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534689536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773494.953101:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773494.953105:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773494.953107:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773494.953114:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2490 00000100:00080000:0.0:1790773494.953118:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773494.953122:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773494/real 1790773494] req@ffff9c4ec7a7f480 x1877759534688896/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773494.953124:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773494.953134:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773494.953154:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534689664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773494.953162:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534688896/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773494.953168:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773494.953177:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773494.953179:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773494.953182:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2490 00000100:00080000:0.0:1790773494.953184:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773494.953192:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773494.953204:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773494.953220:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534689792/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773494.953224:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773494.953230:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773494.953236:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773494.953237:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773494.953245:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773494.953251:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534688896:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773494.953255:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2490 00000100:00080000:0.0:1790773494.953264:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773494.953270:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773494.953276:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534689024:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773494.953279:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773494.953298:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534689920/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773494.953340:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773494.953358:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773494/real 1790773494] req@ffff9c4ec7a7df80 x1877759534689024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773494.953372:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534689024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773494.953400:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773494.953405:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773494.953410:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534689024:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773494.953440:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534689152:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773494.953492:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773494.953504:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773494/real 1790773494] req@ffff9c4ec7a7ed80 x1877759534689152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773494.953516:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534689152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773494.953536:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773494.953539:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773494.953543:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534689152:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773494.953575:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534689280:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773494.953620:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773494.953627:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773494/real 1790773494] req@ffff9c4ec7a7d880 x1877759534689280/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773494.953636:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534689280/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773494.953643:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773494.953646:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773494.953649:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534689280:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773494.953654:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534689408:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773494.953665:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773494.953673:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773494/real 1790773494] req@ffff9c4ec7a7c380 x1877759534689408/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773494.953704:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534689408/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773494.953712:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773494.953715:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773494.953718:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534689408:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773494.953723:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534689536:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773494.953737:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773494.953756:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773494/real 1790773494] req@ffff9c4ec7a7ca80 x1877759534689536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773494.953765:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534689536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773494.953778:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773494.953780:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773494.953783:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534689536:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773494.953787:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534689664:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773494.953799:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773494.953805:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773494/real 1790773494] req@ffff9c4ec7a7fb80 x1877759534689664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773494.953813:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534689664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773494.953820:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773494.953823:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773494.953826:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534689664:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773494.953830:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534689792:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773494.953841:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773494.953848:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773494/real 1790773494] req@ffff9c4ec7a7e300 x1877759534689792/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773494.953855:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534689792/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773494.953863:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773494.953865:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773494.953868:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534689792:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773494.953873:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534689920:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773494.953884:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773494.953891:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773494/real 1790773494] req@ffff9c4ec7a7c700 x1877759534689920/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773494.953899:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534689920/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773494.953906:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773494.953909:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773494.953912:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534689920:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773495.016794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773495.016805:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773495.016809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773495.016809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773495.080777:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773495.080816:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773495.080865:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773495.208739:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773495.208739:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773495.272803:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773495.336752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773495.400732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773495.400738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773495.400740:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773495.464702:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773495.528772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773495.528789:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773495.528791:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773495.656788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773495.656877:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773495.656893:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773495.656895:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773495.720878:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773495.720881:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773495.720885:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773495.720944:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773495.720951:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773495.720967:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773495.848853:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773495.912780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773495.976798:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773496.040877:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773496.040878:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773496.040913:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773496.040917:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773496.104791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773496.104801:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773496.104973:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773496.232754:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773496.232798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773496.296779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773496.360745:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773496.424766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773496.424766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773496.424772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773496.488802:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773496.552804:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773496.552806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773496.552892:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773496.680752:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773496.680799:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773496.680826:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773496.680834:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773496.744744:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773496.744788:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773496.744828:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773496.744839:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773496.744847:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773496.744848:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773496.872799:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773496.936768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773497.000798:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773497.064790:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773497.064805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773497.064834:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773497.064843:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773497.128861:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773497.128864:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773497.129008:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773497.256780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773497.256808:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773497.320751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773497.384790:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773497.448787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773497.448805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773497.448818:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773497.512779:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773497.576746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773497.576760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773497.576791:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773497.704777:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773497.704847:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773497.704888:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773497.704909:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773497.768778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773497.768793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773497.768828:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773497.768834:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773497.768878:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773497.768902:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773497.896763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773497.960747:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773498.024771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773498.088850:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773498.088856:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773498.088907:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773498.088908:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773498.152802:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773498.152866:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773498.153005:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773498.280750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773498.280802:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773498.344743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773498.408793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773498.472756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773498.472772:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773498.472783:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773498.536731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773498.600755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773498.600764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773498.600770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773498.728774:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773498.728793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773498.728825:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773498.728853:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773498.792753:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773498.792758:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773498.792804:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773498.792809:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773498.792828:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773498.792828:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773498.920756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773498.984738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773499.048758:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773499.112761:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773499.112793:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773499.112797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773499.112800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773499.176785:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773499.176796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773499.176904:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773499.304745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773499.304784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773499.368768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773499.432775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773499.496777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773499.496783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773499.496801:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773499.560782:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773499.624768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773499.624783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773499.624784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773499.752770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773499.752776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773499.752787:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773499.752824:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773499.816781:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773499.816793:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773499.816816:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773499.816836:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773499.816864:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773499.816881:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773499.944725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773500.008715:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773500.072794:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773500.072809:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773500.072810:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773500.072818:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773500.072834:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2495 00000100:00080000:3.0:1790773500.072848:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773500.072865:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773500.072945:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534690048/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773500.072985:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773500.072992:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773500.073003:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773500.073011:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2495 00000100:00080000:3.0:1790773500.073017:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773500.073024:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773500.073035:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773500.073054:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534690176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773500.073067:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534690048:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773500.073070:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773500.073082:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773500.073084:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773500.073090:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2495 00000100:00080000:3.0:1790773500.073094:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773500.073099:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773500.073105:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773500.073120:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534690304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773500.073132:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773500.073137:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773500.073139:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773500.073143:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2495 00000100:00080000:3.0:1790773500.073148:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773500.073154:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773500.073157:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773500.073164:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773500.073177:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534690432/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773500.073188:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773500.073192:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773500.073194:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773500.073200:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2495 00000100:00080000:3.0:1790773500.073205:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773500.073209:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773500/real 1790773500] req@ffff9c4ef73c1c00 x1877759534690048/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773500.073214:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773500.073222:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773500.073241:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534690560/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773500.073247:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534690048/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773500.073261:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773500.073271:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773500.073273:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773500.073274:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773500.073281:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2495 00000100:00080000:1.0:1790773500.073287:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773500.073289:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773500.073294:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534690048:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773500.073313:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773500.073373:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773500.073389:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534690176:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773500.073396:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534690688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773500.073415:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773500.073420:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:1.0:1790773500.073423:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773500.073424:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773500.073434:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2495 00000100:00080000:3.0:1790773500.073441:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773500.073446:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773500/real 1790773500] req@ffff9c4ef73c2300 x1877759534690176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773500.073451:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773500.073462:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773500.073470:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534690176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773500.073480:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534690816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773500.073494:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773500.073501:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773500.073510:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534690176:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773500.073513:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773500.073520:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534690304:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773500.073522:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773500.073530:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773500.073540:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2495 00000800:00000100:1.0:1790773500.073548:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773500.073548:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773500.073563:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773500.073566:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773500/real 1790773500] req@ffff9c4ef73c0e00 x1877759534690304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773500.073571:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773500.073586:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534690304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773500.073593:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534690944/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773500.073606:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773500.073617:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773500.073627:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534690304:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773500.073632:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773500.073638:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773500.073639:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534690432:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790773500.073643:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773500.073677:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2495 00000100:00080000:3.0:1790773500.073683:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773500.073689:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773500.073696:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:1.0:1790773500.073707:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773500.073713:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534691072/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773500.073725:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773500/real 1790773500] req@ffff9c4ef73c0a80 x1877759534690432/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773500.073739:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534690432/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773500.073750:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773500.073754:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773500.073758:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534690432:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773500.073764:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534690560:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773500.073780:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773500.073788:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773500/real 1790773500] req@ffff9c4ef73c3800 x1877759534690560/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773500.073799:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534690560/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773500.073807:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773500.073810:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773500.073813:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534690560:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773500.073830:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534690688:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773500.073845:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773500.073854:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773500/real 1790773500] req@ffff9c4ef73c2d80 x1877759534690688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773500.073865:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534690688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773500.073879:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773500.073882:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773500.073885:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534690688:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773500.073891:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534690816:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773500.073910:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773500.073926:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773500/real 1790773500] req@ffff9c4ef73c2a00 x1877759534690816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773500.073944:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534690816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773500.073957:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773500.073962:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773500.073968:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534690816:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773500.073977:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534690944:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773500.074012:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773500.074023:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773500/real 1790773500] req@ffff9c4ef73c0700 x1877759534690944/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773500.074034:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534690944/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773500.074045:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773500.074048:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773500.074051:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534690944:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773500.074057:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534691072:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773500.074071:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773500.074079:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773500/real 1790773500] req@ffff9c4ef73c1500 x1877759534691072/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773500.074089:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534691072/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773500.074099:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773500.074102:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773500.074105:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534691072:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773500.136735:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773500.136749:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773500.136764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773500.136764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773500.200746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773500.200759:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773500.200888:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773500.328730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773500.328755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773500.392723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773500.456721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773500.520804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773500.520804:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773500.520807:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773500.584756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773500.648768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773500.648769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773500.648773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773500.776761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773500.776768:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773500.776785:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773500.776803:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773500.840765:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773500.840799:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773500.840808:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773500.840817:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773500.840841:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773500.840864:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773500.968738:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773501.032746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773501.096787:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773501.160775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773501.160799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773501.160799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773501.160803:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773501.224751:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773501.224793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773501.224832:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773501.352749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773501.352769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773501.416786:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773501.480779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773501.544770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773501.544779:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773501.544803:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773501.608877:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773501.672769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773501.672770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773501.672770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773501.800759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773501.800773:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773501.800780:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773501.800793:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773501.864786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773501.864793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773501.864819:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773501.864834:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773501.864855:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773501.864871:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773501.992801:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773502.056758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773502.120754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773502.184768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773502.184781:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773502.184792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773502.184805:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773502.248775:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773502.248799:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773502.248844:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773502.376750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773502.376756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773502.440796:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773502.504758:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773502.568775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773502.568775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773502.568776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773502.632715:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773502.696716:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773502.696724:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773502.696738:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773502.824812:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773502.824818:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773502.824821:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773502.824834:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773502.888779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773502.888804:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773502.888806:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773502.888819:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773502.888841:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773502.888851:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773503.016709:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773503.080793:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773503.144772:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773503.208746:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773503.208751:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773503.208762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773503.208801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773503.272805:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773503.272811:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773503.272944:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773503.400779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773503.400785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773503.464726:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773503.528732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773503.592723:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773503.592723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773503.592723:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773503.656707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773503.720774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773503.720774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773503.720774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773503.848734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773503.848756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773503.848757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773503.848780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773503.912783:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773503.912786:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773503.912835:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773503.912842:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773503.912877:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773503.912895:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773504.040738:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773504.104755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773504.168795:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773504.232732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773504.232732:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773504.232788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773504.232794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773504.296752:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773504.296760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773504.296959:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773504.424763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773504.424779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773504.488754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773504.552792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773504.616773:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773504.616800:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773504.616815:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773504.680795:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773504.744774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773504.744794:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773504.744796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773504.872815:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773504.872822:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773504.872834:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773504.872834:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773504.936805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773504.936819:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773504.936868:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773504.936877:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773504.936899:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773504.936935:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773505.064760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773505.128753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773505.192786:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773505.192816:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773505.192827:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773505.192833:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773505.192846:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2501 00000100:00080000:2.0:1790773505.192858:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773505.192872:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773505.192918:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534691200/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773505.192955:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773505.192959:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773505.192962:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773505.192967:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2501 00000100:00080000:2.0:1790773505.192971:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773505.192976:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773505.192987:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773505.193004:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534691328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773505.193028:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773505.193033:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773505.193037:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773505.193042:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2501 00000100:00080000:2.0:1790773505.193046:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773505.193057:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534691200:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773505.193060:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773505.193082:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773505.193099:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534691456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773505.193128:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773505.193134:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773505.193138:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773505.193146:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2501 00000100:00080000:2.0:1790773505.193154:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773505.193164:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773505.193175:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773505.193197:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534691584/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773505.193210:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773505.193240:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773505.193244:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773505.193262:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773505.193263:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2501 00000100:00080000:2.0:1790773505.193278:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773505.193292:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773505.193321:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773505.193348:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534691712/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773505.193362:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773505/real 1790773505] req@ffff9c4efbcc0700 x1877759534691200/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773505.193419:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534691200/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773505.193424:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773505.193450:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773505.193459:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773505.193459:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773505.193481:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773505.193489:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2501 00000100:00100000:1.0:1790773505.193491:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534691200:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773505.193511:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773505.193518:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534691328:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773505.193532:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773505.193544:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773505.193578:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773505.193607:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534691840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773505.193608:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773505/real 1790773505] req@ffff9c4efbcc2680 x1877759534691328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773505.193664:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773505.193670:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773505.193674:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773505.193685:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2501 00000100:00080000:2.0:1790773505.193708:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773505.193727:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534691328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773505.193758:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773505.193766:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773505.193775:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773505.193781:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534691328:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790773505.193792:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773505.193814:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534691968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773505.193826:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773505.193831:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773505.193834:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773505.193841:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2501 00000100:00080000:2.0:1790773505.193845:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773505.193853:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534691456:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773505.193855:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773505.193862:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773505.193873:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534692096/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773505.193907:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773505.193913:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773505.193916:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773505.193923:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2501 00000100:00080000:2.0:1790773505.193934:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773505.193939:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773505.193946:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:1.0:1790773505.193958:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773505.193973:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534692224/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773505.193988:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773505/real 1790773505] req@ffff9c4efbcc0a80 x1877759534691456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773505.194009:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534691456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773505.194025:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773505.194031:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773505.194036:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534691456:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773505.194046:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534691584:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773505.194072:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773505.194087:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773505/real 1790773505] req@ffff9c4efbcc2300 x1877759534691584/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773505.194111:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534691584/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773505.194132:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773505.194139:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773505.194147:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534691584:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773505.194161:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534691712:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773505.194201:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773505.194244:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773505/real 1790773505] req@ffff9c4efbcc1880 x1877759534691712/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773505.194262:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534691712/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773505.194275:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773505.194280:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773505.194285:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534691712:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773505.194294:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534691840:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773505.194335:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773505.194350:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773505/real 1790773505] req@ffff9c4efbcc1500 x1877759534691840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773505.194364:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534691840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773505.194383:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773505.194388:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773505.194393:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534691840:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773505.194401:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534691968:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773505.194421:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773505.194434:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773505/real 1790773505] req@ffff9c4efbcc1f80 x1877759534691968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773505.194448:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534691968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773505.194459:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773505.194463:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773505.194467:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534691968:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773505.194496:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534692096:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773505.194518:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773505.194531:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773505/real 1790773505] req@ffff9c4efbcc1c00 x1877759534692096/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773505.194545:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534692096/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773505.194568:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773505.194572:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773505.194577:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534692096:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773505.194584:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534692224:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773505.194603:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773505.194616:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773505/real 1790773505] req@ffff9c4efbcc2a00 x1877759534692224/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773505.194641:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534692224/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773505.194687:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773505.194701:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773505.194706:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534692224:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773505.256790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773505.256847:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773505.256854:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773505.256854:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773505.320819:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773505.320829:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773505.320994:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773505.448749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773505.448795:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773505.512798:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773505.576774:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773505.640786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773505.640836:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773505.640846:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773505.704787:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773505.768831:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773505.768831:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773505.768833:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773505.896768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773505.896776:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773505.896796:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773505.896839:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773505.960717:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773505.960765:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773505.960798:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773505.960803:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773505.960806:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773505.960811:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773506.088736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773506.152802:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773506.216761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773506.280788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773506.280797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773506.280857:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773506.280870:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773506.344719:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773506.344782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773506.344960:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773506.472721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773506.472796:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773506.536782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773506.600799:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773506.664771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773506.664790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773506.664790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773506.728759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773506.792745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773506.792778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773506.792841:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773506.920799:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773506.920804:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773506.920807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773506.920813:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773506.984800:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773506.984803:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773506.984820:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773506.984854:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773506.984861:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773506.984906:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773507.112751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773507.176761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773507.240756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773507.304774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773507.304786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773507.304808:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773507.304817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773507.368763:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773507.368791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773507.368885:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773507.496765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773507.496775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773507.560774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773507.624755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773507.688807:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773507.688807:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773507.688819:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773507.752750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773507.816797:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773507.816801:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773507.816805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773507.944757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773507.944765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773507.944800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773507.944829:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773508.008772:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773508.008774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773508.008815:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773508.008823:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773508.008871:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773508.008872:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773508.136752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773508.200736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773508.264766:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773508.328744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773508.328745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773508.328785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773508.328799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773508.392738:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773508.392742:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773508.392842:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773508.520731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773508.520731:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773508.584739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773508.648794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773508.712770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773508.712801:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773508.712956:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773508.776885:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773508.840782:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773508.840783:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773508.840792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773508.968826:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773508.968826:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773508.968836:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773508.968847:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773509.032796:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773509.032809:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773509.032817:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773509.032868:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773509.032871:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773509.032874:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773509.160774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773509.224770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773509.288725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773509.352723:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773509.352723:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773509.352738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773509.352766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773509.416732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773509.416801:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773509.416835:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773509.544709:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773509.544739:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773509.608786:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773509.672785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773509.736796:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773509.736796:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773509.736842:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773509.800754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773509.864786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773509.864789:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773509.864791:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773509.992753:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773509.992753:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773509.992789:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773509.992791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773510.056787:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773510.056803:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773510.056811:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773510.056824:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773510.056836:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773510.056856:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773510.184754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773510.248843:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773510.312758:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773510.312762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773510.312770:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773510.312776:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773510.312787:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2506 00000100:00080000:2.0:1790773510.312797:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773510.312819:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773510.312852:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534692352/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773510.312874:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773510.312877:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773510.312879:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773510.312882:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2506 00000100:00080000:2.0:1790773510.312885:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773510.312888:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773510.312897:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773510.312908:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534692480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773510.312927:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773510.312930:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773510.312931:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773510.312934:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2506 00000100:00080000:2.0:1790773510.312937:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773510.312939:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773510.312943:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773510.312959:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534692352:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773510.312959:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534692608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773510.312986:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773510.312990:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773510.312992:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773510.312998:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2506 00000100:00080000:2.0:1790773510.313003:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773510.313008:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773510.313015:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773510.313033:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534692736/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773510.313043:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773510.313047:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773510.313049:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773510.313053:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2506 00000800:00000100:1.0:1790773510.313059:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773510.313062:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773510.313072:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773510.313077:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773510.313093:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534692864/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773510.313130:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773510.313135:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773510.313137:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773510.313143:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2506 00000100:00080000:2.0:1790773510.313151:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773510.313157:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773510.313165:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773510/real 1790773510] req@ffff9c4efbcc0380 x1877759534692352/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773510.313168:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773510.313198:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534692992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773510.313203:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534692352/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773510.313271:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773510.313271:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773510.313285:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773510.313288:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773510.313290:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773510.313295:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534692352:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773510.313302:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2506 00000100:00080000:2.0:1790773510.313308:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773510.313312:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534692480:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773510.313320:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773510.313328:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773510.313340:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773510.313348:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534693120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773510.313354:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773510/real 1790773510] req@ffff9c4efbcc3100 x1877759534692480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773510.313412:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773510.313414:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534692480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773510.313421:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773510.313429:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773510.313441:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2506 00000100:00080000:2.0:1790773510.313446:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773510.313453:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773510.313455:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790773510.313459:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773510.313479:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773510.313487:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534692480:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773510.313488:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534693248/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773510.313515:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773510.313518:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773510.313520:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773510.313524:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2506 00000100:00080000:2.0:1790773510.313526:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773510.313528:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773510.313542:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773510.313561:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534692608:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773510.313562:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534693376/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773510.313616:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773510.313629:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773510/real 1790773510] req@ffff9c4efbcc2a00 x1877759534692608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773510.313640:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534692608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773510.313676:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773510.313680:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773510.313684:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534692608:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773510.313690:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534692736:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773510.313710:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773510.313720:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773510/real 1790773510] req@ffff9c4efbcc1c00 x1877759534692736/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773510.313747:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534692736/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773510.313755:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773510.313757:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773510.313760:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534692736:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773510.313765:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534692864:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773510.313778:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773510.313784:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773510/real 1790773510] req@ffff9c4efbcc1f80 x1877759534692864/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773510.313791:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534692864/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773510.313817:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773510.313820:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773510.313822:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534692864:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773510.313828:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534692992:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773510.313844:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773510.313852:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773510/real 1790773510] req@ffff9c4efbcc1500 x1877759534692992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773510.313868:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534692992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773510.313882:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773510.313884:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773510.313887:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534692992:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773510.313892:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534693120:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773510.313905:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773510.313913:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773510/real 1790773510] req@ffff9c4efbcc1880 x1877759534693120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773510.313922:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534693120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773510.313929:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773510.313932:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773510.313934:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534693120:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773510.313939:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534693248:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773510.313952:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773510.313960:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773510/real 1790773510] req@ffff9c4efbcc2300 x1877759534693248/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773510.313969:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534693248/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773510.313983:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773510.313986:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773510.313989:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534693248:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773510.314012:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534693376:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773510.314026:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773510.314034:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773510/real 1790773510] req@ffff9c4efbcc0a80 x1877759534693376/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773510.314043:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534693376/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773510.314052:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773510.314055:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773510.314057:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534693376:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773510.376787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773510.376796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773510.376819:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773510.376820:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773510.440754:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773510.440799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773510.440845:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773510.568736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773510.568787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773510.632744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773510.696787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773510.760737:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773510.760742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773510.760794:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773510.824780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773510.888769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773510.888790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773510.888828:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773511.016768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773511.016801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773511.016809:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773511.016816:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773511.080834:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773511.080849:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773511.080849:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773511.080860:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773511.080921:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773511.080924:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773511.208760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773511.272801:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773511.336776:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773511.400765:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773511.400816:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773511.400817:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773511.400819:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773511.464791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773511.464893:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773511.464927:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773511.592785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773511.592789:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773511.656790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773511.720835:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773511.784846:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773511.784847:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773511.784852:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773511.848737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773511.912759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773511.912799:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773511.912803:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773512.040790:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773512.040842:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773512.040856:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773512.040858:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773512.104781:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773512.104792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773512.104802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773512.104802:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773512.104853:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773512.104863:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773512.232738:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773512.296749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773512.360773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773512.424782:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773512.424792:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773512.424799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773512.424806:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773512.488804:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773512.488815:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773512.488947:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773512.616746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773512.616789:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773512.680757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773512.744763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773512.808729:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773512.808732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773512.808755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773512.872830:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773512.936776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773512.936780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773512.936789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773513.064788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773513.064803:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773513.064804:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773513.064812:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773513.128796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773513.128809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773513.128814:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773513.128817:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773513.128854:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773513.128865:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773513.256715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773513.320787:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773513.384731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773513.448749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773513.448760:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773513.448792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773513.448794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773513.512729:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773513.512763:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773513.512784:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773513.640766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773513.640798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773513.704745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773513.768734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773513.832715:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773513.832722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773513.832737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773513.896723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773513.960710:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773513.960731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773513.960731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773514.088715:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773514.088726:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773514.088726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773514.088727:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773514.152711:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773514.152715:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773514.152731:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773514.152735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773514.152757:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773514.152758:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773514.280701:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773514.344729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773514.408708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773514.472727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773514.472740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773514.472742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773514.472742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773514.536740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773514.536749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773514.536780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773514.664767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773514.664781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773514.728739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773514.792737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773514.856736:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773514.856799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773514.856841:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773514.920769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773514.984771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773514.984810:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773514.984821:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773515.112760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773515.112804:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773515.112809:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773515.112818:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773515.176758:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773515.176783:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773515.176786:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773515.176792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773515.176820:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773515.176830:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773515.304731:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773515.368788:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773515.432749:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773515.432827:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773515.432836:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773515.432841:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773515.432852:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2511 00000100:00080000:3.0:1790773515.432864:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773515.432878:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773515.432924:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534693504/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773515.432960:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773515.432965:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773515.432967:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773515.432972:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2511 00000100:00080000:3.0:1790773515.432976:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773515.432982:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773515.432993:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773515.433012:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534693632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773515.433044:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534693504:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773515.433050:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773515.433058:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773515.433060:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773515.433067:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2511 00000100:00080000:3.0:1790773515.433074:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773515.433081:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773515.433088:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773515.433106:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534693760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773515.433118:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773515.433122:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773515.433124:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773515.433132:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773515.433138:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2511 00000100:00080000:3.0:1790773515.433147:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773515.433155:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773515.433163:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773515.433178:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534693888/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773515.433189:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773515.433194:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773515/real 1790773515] req@ffff9c4ef73c0000 x1877759534693504/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773515.433195:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773515.433202:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773515.433214:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2511 00000100:00080000:3.0:1790773515.433218:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773515.433223:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773515.433227:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773515.433244:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534694016/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773515.433248:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534693504/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773515.433284:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773515.433289:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773515.433293:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773515.433302:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773515.433306:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773515.433315:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2511 00000100:00100000:1.0:1790773515.433322:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534693504:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773515.433327:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773515.433346:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773515.433355:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773515.433373:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534693632:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773515.433380:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534694144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773515.433405:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773515.433410:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773515.433412:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773515.433419:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773515.433427:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2511 00000100:00080000:3.0:1790773515.433435:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773515.433444:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773515.433448:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773515/real 1790773515] req@ffff9c4ef73c1500 x1877759534693632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773515.433453:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773515.433478:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534694272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773515.433490:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773515.433492:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534693632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773515.433499:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773515.433505:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773515.433521:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2511 00000100:00080000:3.0:1790773515.433528:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773515.433535:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773515.433541:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773515.433542:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773515.433555:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790773515.433562:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534694400/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773515.433564:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534693632:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773515.433589:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773515.433590:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534693760:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773515.433598:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773515.433606:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773515.433613:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2511 00000100:00080000:3.0:1790773515.433618:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773515.433623:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773515.433630:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773515.433687:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534694528/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773515.433699:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773515.433731:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773515/real 1790773515] req@ffff9c4ef73c0700 x1877759534693760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773515.433750:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534693760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773515.433770:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773515.433775:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773515.433779:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534693760:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773515.433787:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534693888:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773515.433825:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773515.433838:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773515/real 1790773515] req@ffff9c4ef73c2a00 x1877759534693888/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773515.433852:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534693888/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773515.433865:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773515.433869:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773515.433873:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534693888:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773515.433881:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534694016:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773515.433898:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773515.433908:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773515/real 1790773515] req@ffff9c4ef73c2d80 x1877759534694016/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773515.433922:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534694016/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773515.433932:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773515.433936:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773515.433941:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534694016:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773515.433964:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534694144:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773515.433983:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773515.433994:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773515/real 1790773515] req@ffff9c4ef73c3800 x1877759534694144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773515.434007:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534694144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773515.434049:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773515.434054:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773515.434059:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534694144:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773515.434066:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534694272:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773515.434091:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773515.434105:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773515/real 1790773515] req@ffff9c4ef73c0a80 x1877759534694272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773515.434119:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534694272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773515.434128:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773515.434132:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773515.434136:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534694272:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773515.434142:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534694400:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773515.434158:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773515.434170:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773515/real 1790773515] req@ffff9c4ef73c0e00 x1877759534694400/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773515.434183:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534694400/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773515.434195:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773515.434198:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773515.434202:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534694400:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773515.434208:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534694528:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773515.434225:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773515.434234:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773515/real 1790773515] req@ffff9c4ef73c2300 x1877759534694528/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773515.434247:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534694528/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773515.434258:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773515.434262:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773515.434265:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534694528:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773515.496784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773515.496802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773515.496829:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773515.496869:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773515.560755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773515.560774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773515.560837:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773515.688768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773515.688787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773515.752776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773515.816801:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773515.880764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773515.880785:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773515.880802:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773515.944757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773516.008722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773516.008771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773516.008778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773516.136754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773516.136758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773516.136765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773516.136767:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773516.200760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773516.200796:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773516.200805:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773516.200811:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773516.200832:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773516.200885:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773516.328805:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773516.392751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773516.456759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773516.520773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773516.520797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773516.520828:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773516.520890:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773516.584825:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773516.584836:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773516.584977:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773516.712796:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773516.712814:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773516.776756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773516.840820:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773516.904793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773516.904793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773516.904828:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773516.968727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773517.032774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773517.032774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773517.032805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773517.160766:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773517.160771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773517.160789:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773517.160790:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773517.224795:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773517.224842:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773517.224846:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773517.224858:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773517.224860:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773517.224900:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773517.352764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773517.416761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773517.480845:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773517.544799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773517.544848:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773517.544854:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773517.544854:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773517.608831:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773517.608839:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773517.608869:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773517.736763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773517.736792:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773517.800859:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773517.864815:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773517.928800:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773517.928873:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773517.928882:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773517.992789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773518.056795:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773518.056799:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773518.056801:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773518.184776:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773518.184801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773518.184865:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773518.184872:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773518.248752:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773518.248791:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773518.248802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773518.248811:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773518.248835:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773518.248866:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773518.376773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773518.440748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773518.504724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773518.568772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773518.568772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773518.568773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773518.568775:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773518.632743:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773518.632784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773518.632936:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773518.760732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773518.760781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773518.824791:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773518.888755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773518.952732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773518.952739:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773518.952741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773519.016757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773519.080763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773519.080770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773519.080774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773519.208732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773519.208733:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773519.208737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773519.208739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773519.272768:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773519.272789:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773519.272789:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773519.272819:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773519.272835:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773519.272875:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773519.400729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773519.464731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773519.528801:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773519.592735:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773519.592735:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773519.592739:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773519.592748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773519.656712:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773519.656739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773519.656824:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773519.784768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773519.784771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773519.848805:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773519.912767:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773519.976751:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773519.976791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773519.976842:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773520.040770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773520.104784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773520.104789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773520.104809:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773520.232737:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773520.232767:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773520.232780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773520.232795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773520.296740:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773520.296788:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773520.296798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773520.296823:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773520.296846:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773520.296877:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773520.424742:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773520.488757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773520.552827:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773520.552832:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773520.552849:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773520.552857:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773520.552870:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2516 00000100:00080000:2.0:1790773520.552884:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773520.552898:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773520.552976:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534694656/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773520.553035:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773520.553040:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773520.553042:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773520.553047:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2516 00000100:00080000:2.0:1790773520.553053:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773520.553060:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773520.553077:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773520.553116:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534694784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773520.553129:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773520.553134:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773520.553138:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773520.553143:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2516 00000100:00080000:2.0:1790773520.553148:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773520.553153:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534694656:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773520.553158:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773520.553166:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773520.553179:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534694912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773520.553187:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773520.553190:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773520.553192:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773520.553196:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2516 00000100:00080000:2.0:1790773520.553199:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773520.553202:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773520.553206:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773520.553214:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534695040/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773520.553221:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773520.553224:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773520.553225:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773520.553228:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2516 00000100:00080000:2.0:1790773520.553230:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773520.553233:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773520.553237:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773520.553245:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534695168/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773520.553252:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773520.553256:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773520.553257:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773520.553260:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2516 00000100:00080000:2.0:1790773520.553262:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773520.553265:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773520.553269:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773520.553280:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534695296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773520.553284:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534694784:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773520.553295:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773520.553298:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773520.553302:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773520.553307:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2516 00000100:00080000:2.0:1790773520.553310:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773520.553314:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773520.553318:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773520.553329:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534695424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773520.553337:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534694912:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773520.553338:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773520.553343:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773520.553345:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773520.553358:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2516 00000100:00080000:2.0:1790773520.553368:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773520.553373:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773520.553380:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773520.553381:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534695040:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773520.553401:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534695552/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773520.553408:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773520.553412:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773520.553413:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534695168:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773520.553414:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773520.553422:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2516 00000100:00080000:2.0:1790773520.553425:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773520.553430:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773520.553433:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773520.553440:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534695296:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773520.553444:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534695680/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773520.553478:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534695424:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773520.553496:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534695552:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773520.553509:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534695680:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773520.616746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773520.616815:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773520.616827:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773520.616842:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773520.680739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773520.680786:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773520.680880:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773520.808731:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773520.808735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773520.872768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773520.936741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773521.000767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773521.000788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773521.000817:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773521.064752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773521.128801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773521.128801:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773521.128821:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773521.256738:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773521.256751:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773521.256771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773521.256789:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773521.320731:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773521.320734:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773521.320746:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773521.320766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773521.320801:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773521.320801:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773521.448819:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773521.512774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773521.576858:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773521.640746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773521.640747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773521.640759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773521.640760:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773521.704764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773521.704776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773521.704786:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773521.832723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773521.832754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773521.896738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773521.960768:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773522.024804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773522.024804:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773522.024806:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773522.088818:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773522.152716:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773522.152721:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773522.152789:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773522.280739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773522.280742:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773522.280744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773522.280784:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773522.344787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773522.344804:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773522.344805:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773522.344806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773522.344846:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773522.344860:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773522.472724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773522.536744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773522.600807:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773522.664781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773522.664781:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773522.664782:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773522.664782:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773522.728799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773522.728818:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773522.728857:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773522.856738:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773522.856752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773522.920776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773522.984783:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773523.048888:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773523.048891:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773523.048892:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773523.112742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773523.176750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773523.176750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773523.176815:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773523.304774:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773523.304774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773523.304784:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773523.304860:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773523.368770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773523.368777:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773523.368810:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773523.368818:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773523.368856:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773523.368890:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773523.496756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773523.560750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773523.625015:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:1.0:1790773523.625186:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:1.0:1790773523.625244:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1790773523.625266:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773523.625348:0:2383:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:1.0:1790773523.625372:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773523.625392:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773523.625437:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773523.625440:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773520/real 1790773523] req@ffff9c4efbcc2a00 x1877759534695680/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790773523.625455:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773523.625474:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773523.625488:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534695680/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1790773523.625496:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:1.0:1790773523.625521:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:0.0:1790773523.625535:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773523.625545:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773523.625552:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534695680:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773523.625562:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773520/real 1790773523] req@ffff9c4efbcc0a80 x1877759534694912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773523.625581:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534694912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773523.625592:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773523.625598:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773523.625603:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534694912:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773523.625612:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773520/real 1790773523] req@ffff9c4efbcc2300 x1877759534695040/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773523.625628:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534695040/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773523.625642:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773523.625647:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773523.625651:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534695040:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773523.625662:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773520/real 1790773523] req@ffff9c4efbcc1880 x1877759534695168/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773523.625743:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534695168/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773523.625756:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773523.625761:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773523.625766:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534695168:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773523.625775:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773520/real 1790773523] req@ffff9c4efbcc1500 x1877759534695296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773523.625791:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534695296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773523.625811:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773523.625817:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773523.625822:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534695296:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773523.625831:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773520/real 1790773523] req@ffff9c4efbcc2680 x1877759534694656/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773523.625847:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534694656/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773523.625875:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773523.625882:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773523.625887:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534694656:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773523.625899:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773520/real 1790773523] req@ffff9c4efbcc0700 x1877759534694784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773523.625915:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534694784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773523.625935:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773523.625940:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773523.625945:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534694784:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773523.625990:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773520/real 1790773523] req@ffff9c4efbcc1f80 x1877759534695424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773523.626005:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534695424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773523.626016:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773523.626028:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773523.626033:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534695424:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773523.626041:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773520/real 1790773523] req@ffff9c4efbcc1c00 x1877759534695552/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773523.626058:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534695552/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773523.626072:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773523.626078:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773523.626082:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534695552:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773523.688811:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773523.688812:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773523.688821:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773523.688833:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773523.752839:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773523.752893:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773523.753071:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773523.880768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773523.880821:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773523.944714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773524.008766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773524.072775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773524.072812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773524.072876:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773524.136721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773524.200827:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773524.200889:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773524.200894:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773524.328813:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773524.328815:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773524.328882:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773524.328908:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773524.392792:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773524.392874:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773524.392907:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773524.392910:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773524.392912:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773524.392964:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773524.520727:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773524.584748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773524.648900:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773524.712813:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773524.712818:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773524.712872:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773524.712896:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773524.776877:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773524.776943:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773524.776949:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773524.904751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773524.904752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773524.968762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773525.032775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773525.096825:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773525.096852:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773525.096949:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773525.160791:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773525.224773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773525.224807:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773525.224811:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773525.352805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773525.352813:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773525.352844:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773525.352873:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773525.416781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773525.416781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773525.416790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773525.416793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773525.416871:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773525.416874:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773525.544770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773525.608780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773525.672787:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773525.672801:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773525.672811:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773525.672866:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2521 00000100:00100000:0.0:1790773525.672877:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773525.672887:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773525.672913:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773525.672975:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534695808/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773525.673022:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773525.673031:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773525.673036:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773525.673048:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2521 00000100:00080000:3.0:1790773525.673057:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773525.673067:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534695808:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773525.673076:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773525.673099:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773525.673146:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773525.673150:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534695936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773525.673198:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773525.673209:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773525/real 1790773525] req@ffff9c4ef73c1c00 x1877759534695808/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773525.673210:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773525.673220:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773525.673245:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2521 00000100:00100000:0.0:1790773525.673254:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534695808/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773525.673258:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773525.673283:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773525.673304:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773525.673305:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773525.673326:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773525.673335:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534695808:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773525.673350:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534696064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773525.673378:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534695936:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773525.673395:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773525.673407:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773525.673412:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773525.673425:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2521 00000800:00000100:0.0:1790773525.673438:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773525.673441:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773525.673467:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773525/real 1790773525] req@ffff9c4ef73c2300 x1877759534695936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773525.673470:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773525.673493:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773525.673506:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534695936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773525.673536:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534696192/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773525.673545:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773525.673574:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773525.673587:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773525.673590:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534695936:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773525.673604:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773525.673612:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773525.673626:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2521 00000100:00100000:0.0:1790773525.673630:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534696064:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773525.673638:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773525.673710:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773525.673732:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773525/real 1790773525] req@ffff9c4ef73c0e00 x1877759534696064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773525.673749:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773525.673754:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534696064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773525.673776:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:0.0:1790773525.673781:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773525.673796:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773525.673802:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534696064:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773525.673831:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534696192:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773525.673862:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773525.673877:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773525/real 1790773525] req@ffff9c4ef73c0a80 x1877759534696192/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773525.673891:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534696192/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773525.673902:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773525.673906:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773525.673911:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534696192:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773525.674029:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534696320/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773525.674066:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773525.674075:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773525.674079:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773525.674090:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2521 00000100:00080000:3.0:1790773525.674100:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773525.674114:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773525.674127:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773525.674158:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534696448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773525.674179:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773525.674187:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773525.674191:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773525.674200:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2521 00000100:00080000:3.0:1790773525.674209:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773525.674219:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773525.674229:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773525.674257:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534696576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773525.674292:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773525.674299:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773525.674303:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773525.674313:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2521 00000100:00080000:3.0:1790773525.674321:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773525.674332:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773525.674342:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773525.674366:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534696704/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773525.674387:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773525.674394:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773525.674398:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773525.674407:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2521 00000100:00080000:3.0:1790773525.674414:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773525.674424:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773525.674435:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773525.674458:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534696832/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773525.674590:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534696320:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773525.674745:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773525.674794:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773525/real 1790773525] req@ffff9c4ef73c3800 x1877759534696320/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773525.674857:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534696320/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773525.674883:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773525.674893:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773525.674902:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534696320:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773525.674918:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534696448:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773525.674966:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773525.674992:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773525/real 1790773525] req@ffff9c4ef73c2d80 x1877759534696448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773525.675020:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534696448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773525.675049:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773525.675074:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773525.675080:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534696448:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773525.675092:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534696576:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773525.675133:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773525.675156:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773525/real 1790773525] req@ffff9c4ef73c2a00 x1877759534696576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773525.675180:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534696576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773525.675192:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773525.675197:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773525.675202:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534696576:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773525.675210:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534696704:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773525.675240:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773525.675253:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773525/real 1790773525] req@ffff9c4ef73c0700 x1877759534696704/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773525.675267:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534696704/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773525.675302:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773525.675307:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773525.675312:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534696704:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773525.675320:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534696832:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773525.675343:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773525.675356:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773525/real 1790773525] req@ffff9c4ef73c1500 x1877759534696832/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773525.675370:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534696832/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773525.675381:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773525.675385:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773525.675389:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534696832:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773525.736738:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773525.736772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773525.736790:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773525.736860:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773525.800780:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773525.800882:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773525.801026:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773525.928797:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773525.928798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773525.992749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773526.056754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773526.120778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773526.120784:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773526.120807:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773526.184842:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773526.248879:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773526.248880:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773526.248880:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773526.376851:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773526.376903:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773526.376914:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773526.376916:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773526.440769:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773526.440920:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773526.440923:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773526.440934:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773526.440965:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773526.440985:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773526.568841:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773526.632782:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773526.696838:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773526.760727:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773526.760749:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773526.760750:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773526.760826:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773526.824880:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773526.824882:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773526.825084:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773526.952730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773526.952786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773527.016763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773527.080778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773527.144758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773527.144759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773527.144779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773527.208775:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773527.272775:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773527.272776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773527.272781:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773527.400759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773527.400765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773527.400770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773527.400821:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773527.464820:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773527.464836:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773527.464856:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773527.464858:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773527.464908:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773527.464950:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773527.592739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773527.656751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773527.720818:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773527.784750:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773527.784797:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773527.784815:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773527.784823:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773527.848773:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773527.848841:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773527.848970:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773527.976809:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773527.976834:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773528.040742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773528.104731:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773528.168744:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773528.168744:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773528.168749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773528.232726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773528.296728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773528.296733:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773528.296738:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773528.424783:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773528.424784:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773528.424784:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773528.424831:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773528.488749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773528.488755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773528.488793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773528.488797:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773528.488829:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773528.488856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773528.616720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773528.680748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773528.744784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773528.808754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773528.808802:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773528.808802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773528.808804:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773528.872822:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773528.872823:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773528.872916:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773529.000785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773529.000789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773529.064773:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773529.128792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773529.192723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773529.192834:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773529.192834:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773529.256703:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773529.320773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773529.320811:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773529.320819:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773529.448816:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773529.448816:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773529.448821:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773529.448823:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773529.512717:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773529.512738:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773529.512807:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773529.512854:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773529.512857:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773529.512892:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773529.640782:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773529.704751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773529.768783:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773529.832825:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773529.832825:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773529.832847:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773529.832873:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773529.896777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773529.896804:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773529.896879:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773530.024777:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773530.024777:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773530.088718:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773530.152770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773530.216765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773530.216766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773530.216801:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773530.280769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773530.344719:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773530.344769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773530.344790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773530.472799:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773530.472799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773530.472799:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773530.472834:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773530.536712:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773530.536788:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773530.536816:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773530.536844:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773530.536845:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773530.536882:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773530.664819:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773530.728810:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773530.792760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773530.792820:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773530.792837:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773530.792846:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773530.792866:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2526 00000100:00080000:1.0:1790773530.792886:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773530.792905:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773530.792971:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534696960/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773530.793021:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773530.793032:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773530.793036:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773530.793047:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2526 00000100:00080000:1.0:1790773530.793057:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773530.793065:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773530.793073:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534696960:192.168.204.150@tcp:250:kworker.0 00000008:00100000:1.0:1790773530.793084:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773530.793108:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534697088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773530.793126:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773530.793132:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773530.793135:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773530.793142:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2526 00000800:00000100:0.0:1790773530.793157:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773530.793160:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773530.793170:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773530.793179:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773530.793196:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534697216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773530.793209:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773530.793215:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773530.793220:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773530.793224:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773530/real 1790773530] req@ffff9c4efac4fb80 x1877759534696960/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773530.793231:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2526 00000100:00080000:1.0:1790773530.793244:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773530.793258:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773530.793267:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773530.793269:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534696960/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773530.793289:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790773530.793292:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534697344/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773530.793306:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773530.793312:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773530.793321:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1790773530.793323:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534696960:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773530.793325:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773530.793339:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2526 00000100:00080000:1.0:1790773530.793344:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773530.793350:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773530.793356:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773530.793371:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534697472/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773530.793377:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534697088:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773530.793392:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773530.793398:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773530.793402:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773530.793411:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2526 00000100:00080000:1.0:1790773530.793419:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773530.793430:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773530.793431:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773530.793448:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773530.793462:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773530/real 1790773530] req@ffff9c4efac4f480 x1877759534697088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773530.793471:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534697600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773530.793499:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534697088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773530.793508:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773530.793530:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773530.793541:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773530.793546:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773530.793548:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773530.793557:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534697088:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773530.793564:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2526 00000100:00080000:1.0:1790773530.793578:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773530.793579:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534697216:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773530.793592:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773530.793601:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773530.793609:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773530.793623:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534697728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773530.793631:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773530/real 1790773530] req@ffff9c4efac4e680 x1877759534697216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773530.793686:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534697216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773530.793697:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773530.793708:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773530.793710:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773530.793716:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534697216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773530.793744:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534697344:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773530.793748:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773530.793754:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773530.793768:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2526 00000100:00080000:1.0:1790773530.793780:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773530.793787:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773530.793802:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773530.793808:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773530/real 1790773530] req@ffff9c4efac4ea00 x1877759534697344/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773530.793826:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773530.793828:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534697344/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773530.793849:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773530.793854:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773530.793857:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534697344:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773530.793870:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534697856/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773530.793881:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534697472:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773530.793900:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773530.793907:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773530.793911:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773530.793923:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773530.793926:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2526 00000100:00080000:1.0:1790773530.793938:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773530.793949:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773530/real 1790773530] req@ffff9c4efac4f800 x1877759534697472/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773530.793951:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773530.793968:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773530.793978:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534697472/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773530.793993:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534697984/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773530.794001:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773530.794024:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773530.794041:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534697472:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773530.794050:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534697600:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773530.794071:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773530.794081:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773530/real 1790773530] req@ffff9c4efac4c700 x1877759534697600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773530.794090:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534697600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773530.794102:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773530.794104:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773530.794107:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534697600:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773530.794112:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534697728:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773530.794123:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773530.794130:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773530/real 1790773530] req@ffff9c4efac4ed80 x1877759534697728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773530.794138:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534697728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773530.794144:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773530.794146:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773530.794149:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534697728:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773530.794161:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534697856:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773530.794174:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773530.794185:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773530/real 1790773530] req@ffff9c4efac4dc00 x1877759534697856/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773530.794197:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534697856/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773530.794209:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773530.794213:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773530.794217:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534697856:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773530.794224:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534697984:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773530.794242:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773530.794251:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773530/real 1790773530] req@ffff9c4efac4df80 x1877759534697984/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773530.794259:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534697984/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773530.794266:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773530.794269:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773530.794271:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534697984:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773530.856715:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773530.856740:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773530.856753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773530.856826:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773530.920771:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773530.920781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773530.920949:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773531.048763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773531.048763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773531.112767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773531.176784:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773531.240736:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773531.240767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773531.240816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773531.304696:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773531.368755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773531.368758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773531.368811:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773531.496753:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773531.496754:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773531.496788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773531.496792:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773531.560745:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773531.560749:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773531.560763:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773531.560829:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773531.560853:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773531.560912:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773531.688790:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773531.752721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773531.816753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773531.880786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773531.880786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773531.880786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773531.880818:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773531.944772:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773531.944818:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773531.944952:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773532.072783:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773532.072808:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773532.136829:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773532.200814:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773532.264780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773532.264808:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773532.264831:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773532.328739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773532.392783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773532.392784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773532.392804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773532.520803:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773532.520817:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773532.520817:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773532.520829:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773532.584786:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773532.584817:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773532.584865:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773532.584871:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773532.584872:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773532.584900:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773532.712792:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773532.776779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773532.840739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773532.904744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773532.904760:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773532.904805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773532.904822:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773532.968817:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773532.968826:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773532.968972:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773533.096779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773533.096779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773533.160799:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773533.224776:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773533.288743:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773533.288743:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773533.288743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773533.352740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773533.416796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773533.416797:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773533.416797:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773533.544791:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773533.544791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773533.544792:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773533.544793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773533.608754:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773533.608754:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773533.608754:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773533.608801:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773533.608802:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773533.608834:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773533.736737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773533.800807:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773533.864772:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773533.928800:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773533.928800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773533.928800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773533.928800:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773533.992742:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773533.992742:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773533.992796:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773534.120775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773534.120775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773534.184789:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773534.248747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773534.312738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773534.312741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773534.312741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773534.376737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773534.440725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773534.440726:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773534.440727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773534.568724:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773534.568731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773534.568741:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773534.568744:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773534.632752:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773534.632752:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773534.632763:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773534.632769:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773534.632789:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773534.632800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773534.760748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773534.824842:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773534.888755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773534.952781:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773534.952781:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773534.952781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773534.952781:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773535.016762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773535.016771:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773535.016805:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773535.144751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773535.144751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773535.208779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773535.272798:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773535.336790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773535.336791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773535.336811:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773535.400730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773535.464836:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773535.464837:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773535.464838:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773535.592736:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773535.592759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773535.592759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773535.592760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773535.656750:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773535.656763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773535.656775:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773535.656780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773535.656817:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773535.656838:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773535.784752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773535.848721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773535.912727:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773535.912769:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773535.912775:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773535.912779:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773535.912787:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2531 00000100:00080000:3.0:1790773535.912795:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773535.912804:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773535.912838:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534698112/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773535.912869:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773535.912872:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773535.912873:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773535.912876:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2531 00000100:00080000:3.0:1790773535.912879:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773535.912883:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773535.912892:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773535.912902:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534698112:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773535.912905:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534698240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773535.912915:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773535.912917:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773535.912920:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773535.912923:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2531 00000100:00080000:3.0:1790773535.912926:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773535.912929:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773535.912932:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773535.912941:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534698368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773535.912948:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773535.912950:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773535.912952:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773535.912955:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2531 00000100:00080000:3.0:1790773535.912958:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773535.912964:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773535.912965:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790773535.912970:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773535.912980:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534698496/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773535.912987:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773535.912989:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773535.912991:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773535.912994:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2531 00000100:00080000:3.0:1790773535.912996:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773535.913000:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773535.913004:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773535.913008:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773535/real 1790773535] req@ffff9c4ef73c0000 x1877759534698112/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773535.913016:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534698624/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773535.913049:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773535.913053:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773535.913055:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773535.913058:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2531 00000100:00100000:0.0:1790773535.913062:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534698112/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773535.913062:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773535.913069:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773535.913075:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773535.913081:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790773535.913086:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534698752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773535.913089:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773535.913098:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534698112:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773535.913101:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773535.913106:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773535.913108:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773535.913113:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2531 00000100:00080000:3.0:1790773535.913115:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773535.913118:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773535.913122:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773535.913124:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534698240:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773535.913131:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534698880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773535.913141:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773535.913144:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1790773535.913146:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773535.913147:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773535.913155:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2531 00000100:00080000:3.0:1790773535.913158:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773535.913164:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773535.913166:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773535/real 1790773535] req@ffff9c4ef73c1500 x1877759534698240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773535.913169:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773535.913180:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534699008/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773535.913188:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773535.913192:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534698240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773535.913193:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773535.913196:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773535.913203:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2531 00000100:00080000:3.0:1790773535.913207:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773535.913213:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773535.913218:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773535.913227:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534699136/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773535.913228:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773535.913236:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773535.913242:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534698240:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773535.913249:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534698368:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773535.913268:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773535.913276:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773535/real 1790773535] req@ffff9c4ef73c0700 x1877759534698368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773535.913285:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534698368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773535.913297:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773535.913300:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773535.913302:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534698368:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773535.913306:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534698496:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773535.913318:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773535.913324:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773535/real 1790773535] req@ffff9c4ef73c2a00 x1877759534698496/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773535.913331:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534698496/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773535.913338:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773535.913340:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773535.913342:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534698496:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773535.913346:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534698624:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773535.913367:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773535.913373:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773535/real 1790773535] req@ffff9c4ef73c2d80 x1877759534698624/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773535.913380:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534698624/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773535.913387:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773535.913389:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773535.913392:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534698624:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773535.913396:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534698752:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773535.913406:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773535.913412:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773535/real 1790773535] req@ffff9c4ef73c3800 x1877759534698752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773535.913419:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534698752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773535.913431:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773535.913433:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773535.913436:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534698752:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773535.913461:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534698880:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773535.913472:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773535.913479:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773535/real 1790773535] req@ffff9c4ef73c0a80 x1877759534698880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773535.913487:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534698880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773535.913493:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773535.913495:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773535.913497:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534698880:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773535.913501:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534699008:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773535.913519:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773535.913525:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773535/real 1790773535] req@ffff9c4ef73c0e00 x1877759534699008/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773535.913532:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534699008/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773535.913539:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773535.913541:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773535.913544:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534699008:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773535.913547:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534699136:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773535.913557:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773535.913563:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773535/real 1790773535] req@ffff9c4ef73c2300 x1877759534699136/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773535.913570:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534699136/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773535.913577:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773535.913579:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773535.913581:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534699136:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773535.976790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773535.976797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773535.976804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773535.976806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773536.040775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773536.040780:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773536.040937:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773536.168725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773536.168728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773536.232756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773536.296755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773536.360729:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773536.360741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773536.360742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773536.424736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773536.488752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773536.488759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773536.488761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773536.616759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773536.616768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773536.616814:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773536.616822:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773536.680733:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773536.680733:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773536.680733:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773536.680747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773536.680779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773536.680780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773536.808724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773536.872763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773536.936705:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773537.000729:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773537.000729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773537.000735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773537.000735:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773537.064737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773537.064747:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773537.064825:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773537.192717:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773537.192719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773537.256729:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773537.320734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773537.384756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773537.384756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773537.384757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773537.448760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773537.512726:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773537.512726:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773537.512750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773537.640719:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773537.640726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773537.640728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773537.640769:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773537.704738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773537.704746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773537.704765:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773537.704778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773537.704800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773537.704808:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773537.832708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773537.896700:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773537.960759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773538.024755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773538.024755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773538.024758:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773538.024766:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773538.088770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773538.088770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773538.088882:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773538.216705:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773538.216725:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773538.280737:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773538.344747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773538.408746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773538.408746:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773538.408746:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773538.472712:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773538.536743:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773538.536743:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773538.536743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773538.664792:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773538.664793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773538.664793:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773538.664793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773538.728750:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773538.728779:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773538.728804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773538.728809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773538.728837:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773538.728848:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773538.856719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773538.920763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773538.984737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773539.048717:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773539.048721:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773539.048723:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773539.048733:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773539.112722:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773539.112770:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773539.112774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773539.240711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773539.240712:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773539.304734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773539.368729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773539.432709:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773539.432711:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773539.433688:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773539.496721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773539.560721:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773539.560721:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773539.560726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773539.688713:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773539.688715:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773539.688718:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773539.688727:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773539.752751:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773539.752753:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773539.752761:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773539.752761:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773539.752788:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773539.752805:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773539.880707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773539.944697:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773540.008765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773540.072717:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773540.072721:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773540.072757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773540.072757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773540.136782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773540.136799:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773540.136801:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773540.264717:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773540.264717:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773540.328735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773540.392719:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773540.456727:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773540.456731:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773540.456774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773540.520721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773540.584748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773540.584758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773540.584786:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773540.712766:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773540.712800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773540.712800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773540.712809:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773540.776733:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773540.776733:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773540.776736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773540.776777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773540.776794:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773540.776803:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773540.904727:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773540.968720:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773541.032766:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773541.032789:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773541.032795:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773541.032806:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773541.032808:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2536 00000100:00080000:2.0:1790773541.032821:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773541.032837:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773541.032886:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534699264/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773541.032927:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773541.032933:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773541.032936:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773541.032942:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2536 00000100:00080000:2.0:1790773541.032947:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773541.032954:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773541.032966:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773541.032980:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534699264:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773541.032984:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534699392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773541.032997:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773541.033001:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773541.033003:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773541.033007:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2536 00000100:00080000:2.0:1790773541.033010:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773541.033013:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773541.033019:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773541.033030:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534699520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773541.033038:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773541.033042:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773541.033044:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773541.033048:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2536 00000100:00080000:2.0:1790773541.033051:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773541.033054:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773541.033059:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:0.0:1790773541.033062:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773541.033073:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534699648/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773541.033082:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773541.033086:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773541.033087:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773541.033091:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2536 00000100:00080000:2.0:1790773541.033094:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773541.033098:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773541.033105:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773541.033121:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773541/real 1790773541] req@ffff9c4efbcc3100 x1877759534699264/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773541.033123:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534699776/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773541.033166:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773541.033170:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773541.033172:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773541.033175:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534699264/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773541.033180:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2536 00000100:00080000:2.0:1790773541.033188:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773541.033196:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773541.033197:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773541.033225:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773541.033226:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773541.033234:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534699264:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773541.033247:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534699904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773541.033259:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773541.033263:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773541.033266:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773541.033270:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534699392:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773541.033279:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2536 00000100:00080000:2.0:1790773541.033289:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773541.033314:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773541.033317:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773541.033330:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773541.033336:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773541/real 1790773541] req@ffff9c4efbcc0380 x1877759534699392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773541.033358:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534700032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773541.033361:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534699392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773541.033385:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773541.033391:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773541.033395:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773541.033402:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2536 00000100:00080000:2.0:1790773541.033408:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773541.033410:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773541.033416:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773541.033452:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773541.033453:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773541.033462:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534699392:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773541.033468:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534700160/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773541.033473:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534699520:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773541.033491:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773541.033495:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773541.033497:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773541.033502:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2536 00000100:00080000:2.0:1790773541.033506:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773541.033510:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773541.033546:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773541.033552:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773541.033561:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773541/real 1790773541] req@ffff9c4efbcc1c00 x1877759534699520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773541.033568:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534700288/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773541.033585:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534699520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773541.033595:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773541.033598:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773541.033603:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534699520:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773541.033609:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534699648:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773541.033627:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773541.033636:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773541/real 1790773541] req@ffff9c4efbcc1f80 x1877759534699648/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773541.033677:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534699648/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773541.033693:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773541.033697:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773541.033700:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534699648:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773541.033706:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534699776:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773541.033732:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773541.033740:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773541/real 1790773541] req@ffff9c4efbcc0700 x1877759534699776/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773541.033749:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534699776/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773541.033784:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773541.033792:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773541.033795:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534699776:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773541.033802:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534699904:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773541.033823:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773541.033833:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773541/real 1790773541] req@ffff9c4efbcc2680 x1877759534699904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773541.033845:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534699904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773541.033877:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773541.033884:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773541.033890:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534699904:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773541.033934:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534700032:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773541.033953:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773541.033962:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773541/real 1790773541] req@ffff9c4efbcc1500 x1877759534700032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773541.033972:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534700032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773541.033979:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773541.033981:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773541.033984:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534700032:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773541.033989:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534700160:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773541.034000:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773541.034007:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773541/real 1790773541] req@ffff9c4efbcc1880 x1877759534700160/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773541.034016:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534700160/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773541.034023:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773541.034025:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773541.034029:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534700160:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773541.034033:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534700288:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773541.034061:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773541.034072:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773541/real 1790773541] req@ffff9c4efbcc2300 x1877759534700288/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773541.034089:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534700288/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773541.034103:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773541.034109:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773541.034114:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534700288:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773541.096726:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773541.096729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773541.096734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773541.096747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773541.160733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773541.160739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773541.160801:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773541.288762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773541.288783:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773541.352752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773541.416746:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773541.480750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773541.480781:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773541.480828:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773541.544701:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773541.608771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773541.608774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773541.608804:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773541.736739:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773541.736758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773541.736782:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773541.736795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773541.800771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773541.800775:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773541.800792:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773541.800832:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773541.800838:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773541.800876:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773541.928724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773541.992720:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773542.056761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773542.120769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773542.120769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773542.120802:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773542.120804:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773542.184740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773542.184803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773542.184859:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773542.312731:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773542.312735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773542.376756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773542.440753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773542.504717:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773542.504744:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773542.504783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773542.568740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773542.632747:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773542.632773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773542.632801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773542.760765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773542.760819:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773542.760837:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773542.760841:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773542.824759:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773542.824762:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773542.824762:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773542.824769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773542.824809:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773542.824812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773542.952720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773543.016784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773543.080763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773543.144823:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773543.144831:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773543.144831:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773543.144831:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773543.208788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773543.208806:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773543.208807:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773543.336732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773543.336732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773543.400778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773543.464749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773543.528734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773543.528734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773543.528735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773543.592716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773543.656727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773543.656728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773543.656734:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773543.784721:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773543.784721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773543.784722:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773543.784725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773543.848790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773543.848797:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773543.848801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773543.848804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773543.848825:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773543.848842:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773543.976764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773544.040761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773544.104781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773544.168749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773544.168753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773544.168769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773544.168771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773544.232726:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773544.232754:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773544.232835:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773544.360753:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773544.360773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773544.424754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773544.488758:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773544.552735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773544.552741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773544.552781:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773544.616796:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773544.680785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773544.680787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773544.680799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773544.808749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773544.808795:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773544.808808:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773544.808815:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773544.872757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773544.872763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773544.872766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773544.872768:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773544.872798:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773544.872804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773545.000782:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773545.064756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773545.128747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773545.192733:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773545.192740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773545.192832:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773545.192832:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773545.256833:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773545.256838:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773545.256862:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773545.384819:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773545.384820:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773545.448755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773545.512759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773545.576762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773545.576762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773545.576772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773545.640775:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773545.704757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773545.704757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773545.704806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773545.832772:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773545.832789:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773545.832795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773545.832802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773545.896732:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773545.896735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773545.896735:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773545.896773:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773545.896782:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773545.896791:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773546.024749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773546.088769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773546.152732:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773546.152740:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773546.152744:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773546.152753:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2542 00000100:00080000:0.0:1790773546.152763:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773546.152773:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773546.152793:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773546.152809:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534700416/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773546.152829:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773546.152833:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773546.152835:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773546.152839:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2542 00000100:00080000:0.0:1790773546.152841:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773546.152845:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773546.152854:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773546.152858:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534700416:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773546.152869:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534700544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773546.152878:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773546.152881:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773546.152884:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773546.152889:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2542 00000100:00080000:0.0:1790773546.152892:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773546.152896:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773546.152900:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773546.152910:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534700672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773546.152918:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773546.152921:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773546.152923:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773546.152926:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2542 00000100:00080000:0.0:1790773546.152936:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773546.152940:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773546.152944:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:1.0:1790773546.152952:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773546.152953:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534700800/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773546.152963:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773546.152966:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773546.152968:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773546.152971:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2542 00000100:00080000:0.0:1790773546.152974:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773546.152978:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773546.152982:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773546.152993:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534700928/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773546.153000:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773546.153003:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773546.153004:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773546.153008:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2542 00000100:00080000:0.0:1790773546.153011:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773546.153015:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773546.153021:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773546.153029:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773546/real 1790773546] req@ffff9c4ec7a7d180 x1877759534700416/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773546.153030:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534701056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773546.153091:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773546.153095:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1790773546.153097:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534700416/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773546.153099:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773546.153118:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2542 00000100:00080000:0.0:1790773546.153124:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773546.153125:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773546.153130:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773546.153131:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790773546.153147:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773546.153147:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534700416:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773546.153176:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534701184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773546.153188:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773546.153192:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534700544:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773546.153193:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773546.153203:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773546.153209:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2542 00000100:00080000:0.0:1790773546.153212:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773546.153216:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773546.153219:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773546.153228:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534701312/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773546.153235:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773546.153248:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773546.153251:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773546.153252:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773546.153257:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2542 00000100:00100000:1.0:1790773546.153259:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773546/real 1790773546] req@ffff9c4ec7a7c700 x1877759534700544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773546.153262:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773546.153277:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773546.153292:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773546.153298:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534700544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773546.153307:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534701440/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773546.153336:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773546.153342:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773546.153347:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534700544:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773546.153358:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534700672:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773546.153376:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773546.153383:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773546/real 1790773546] req@ffff9c4ec7a7e300 x1877759534700672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773546.153392:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534700672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773546.153403:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773546.153405:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773546.153409:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534700672:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773546.153413:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534700800:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773546.153510:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773546.153519:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773546/real 1790773546] req@ffff9c4ec7a7fb80 x1877759534700800/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773546.153527:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534700800/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773546.153535:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773546.153537:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773546.153540:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534700800:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773546.153545:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534700928:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773546.153558:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773546.153565:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773546/real 1790773546] req@ffff9c4ec7a7ca80 x1877759534700928/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773546.153574:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534700928/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773546.153580:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773546.153582:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773546.153585:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534700928:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773546.153590:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534701056:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773546.153601:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773546.153607:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773546/real 1790773546] req@ffff9c4ec7a7c380 x1877759534701056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773546.153615:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534701056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773546.153628:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773546.153631:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773546.153635:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534701056:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773546.153691:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534701184:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773546.153710:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773546.153719:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773546/real 1790773546] req@ffff9c4ec7a7d880 x1877759534701184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773546.153729:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534701184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773546.153738:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773546.153742:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773546.153746:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534701184:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773546.153753:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534701312:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773546.153774:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773546.153786:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773546/real 1790773546] req@ffff9c4ec7a7ed80 x1877759534701312/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773546.153795:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534701312/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773546.153804:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773546.153806:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773546.153809:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534701312:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773546.153814:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534701440:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773546.153828:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773546.153834:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773546/real 1790773546] req@ffff9c4ec7a7df80 x1877759534701440/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773546.153842:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534701440/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773546.153849:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773546.153851:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773546.153853:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534701440:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773546.216754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773546.216775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773546.216775:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773546.216777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773546.280709:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773546.280735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773546.280847:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773546.408768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773546.408781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773546.472756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773546.536789:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773546.600724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773546.600728:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773546.600730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773546.664718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773546.728744:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773546.728744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773546.728745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773546.856758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773546.856759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773546.856759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773546.856759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773546.920733:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773546.920747:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773546.920748:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773546.920758:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773546.920777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773546.920777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773547.048771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773547.112704:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773547.176726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773547.240715:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773547.240718:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773547.240727:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773547.240750:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773547.304727:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773547.304734:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773547.304801:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773547.432729:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773547.432737:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773547.496726:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773547.560712:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773547.624708:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773547.624720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773547.624726:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773547.688714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773547.752759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773547.752759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773547.752760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773547.880762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773547.880771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773547.880779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773547.880780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773547.944740:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773547.944751:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773547.944768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773547.944776:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773547.944803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773547.944806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773548.072745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773548.136728:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773548.200801:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773548.264707:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773548.264718:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773548.264770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773548.264792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773548.328726:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773548.328777:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773548.328872:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773548.456713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773548.456713:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773548.520742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773548.584771:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773548.648778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773548.648778:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773548.648784:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773548.712732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773548.776752:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773548.776753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773548.776764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773548.904791:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773548.904862:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773548.904869:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773548.904870:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773548.968717:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773548.968741:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773548.968761:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773548.968761:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773548.968765:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773548.968783:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773549.096767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773549.160780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773549.224738:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773549.288724:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773549.288725:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773549.288731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773549.288762:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773549.352722:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773549.352777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773549.352877:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773549.480785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773549.480791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773549.544781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773549.608782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773549.672740:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773549.672753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773549.672753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773549.736764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773549.800761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773549.800774:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773549.800780:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773549.928747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773549.928779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773549.928810:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773549.928810:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773549.992798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773549.992800:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773549.992806:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773549.992832:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773549.992886:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773549.992886:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773550.120738:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773550.184763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773550.248731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773550.312723:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773550.312723:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773550.312751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773550.312779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773550.376723:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773550.376727:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773550.376848:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773550.504717:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773550.504717:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773550.568704:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773550.632753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773550.696760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773550.696763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773550.696764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773550.760719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773550.824724:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773550.824773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773550.824774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773550.952736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773550.952737:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773550.952737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773550.952770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773551.016730:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773551.016735:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773551.016761:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773551.016774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773551.016786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773551.016805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773551.144733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773551.208752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773551.272741:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773551.272751:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773551.272757:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773551.272757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773551.272775:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2547 00000100:00080000:0.0:1790773551.272787:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773551.272800:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773551.272837:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534701568/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773551.272867:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773551.272871:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773551.272874:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773551.272879:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2547 00000100:00080000:0.0:1790773551.272884:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773551.272890:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773551.272909:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773551.272915:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534701568:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773551.272932:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534701696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773551.272947:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773551.272951:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773551.272954:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773551.272959:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2547 00000100:00080000:0.0:1790773551.272963:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773551.272969:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773551.272975:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773551.272990:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534701824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773551.273002:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773551.273012:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1790773551.273013:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773551.273020:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773551.273028:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2547 00000100:00080000:0.0:1790773551.273034:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773551.273042:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773551.273052:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773551.273072:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534701952/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773551.273079:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773551/real 1790773551] req@ffff9c4ec7a7f480 x1877759534701568/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773551.273108:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773551.273113:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773551.273116:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773551.273122:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534701568/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773551.273123:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2547 00000100:00080000:0.0:1790773551.273136:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773551.273151:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773551.273153:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790773551.273159:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:1.0:1790773551.273164:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773551.273174:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534701568:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773551.273182:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534702080/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773551.273195:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773551.273198:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773551.273201:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773551.273208:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2547 00000100:00100000:1.0:1790773551.273208:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534701696:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773551.273218:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773551.273227:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773551.273235:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773551.273247:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773551.273250:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534702208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773551.273266:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773551.273269:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773551/real 1790773551] req@ffff9c4ec7a7df80 x1877759534701696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773551.273273:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773551.273279:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773551.273296:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2547 00000100:00080000:0.0:1790773551.273301:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773551.273308:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773551.273317:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773551.273336:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534701696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773551.273350:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534702336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773551.273360:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773551.273373:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773551.273386:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773551.273387:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534701696:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773551.273396:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773551.273400:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773551.273404:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534701824:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773551.273412:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2547 00000100:00080000:0.0:1790773551.273417:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773551.273424:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773551.273430:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:1.0:1790773551.273441:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773551.273449:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534702464/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773551.273463:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773551/real 1790773551] req@ffff9c4ec7a7ed80 x1877759534701824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773551.273464:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773551.273477:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773551.273483:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773551.273498:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2547 00000100:00100000:1.0:1790773551.273503:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534701824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773551.273506:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773551.273518:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773551.273529:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790773551.273531:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773551.273538:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773551.273545:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534701824:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773551.273552:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534702592/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773551.273560:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534701952:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773551.273599:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773551.273614:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773551/real 1790773551] req@ffff9c4ec7a7d880 x1877759534701952/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773551.273627:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534701952/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773551.273639:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773551.273643:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773551.273647:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534701952:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773551.273680:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534702080:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773551.273700:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773551.273712:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773551/real 1790773551] req@ffff9c4ec7a7c380 x1877759534702080/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773551.273724:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534702080/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773551.273736:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773551.273740:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773551.273743:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534702080:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773551.273762:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534702208:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773551.273781:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773551.273792:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773551/real 1790773551] req@ffff9c4ec7a7ca80 x1877759534702208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773551.273805:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534702208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773551.273821:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773551.273825:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773551.273829:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534702208:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773551.273836:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534702336:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773551.273854:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773551.273866:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773551/real 1790773551] req@ffff9c4ec7a7fb80 x1877759534702336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773551.273878:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534702336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773551.273888:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773551.273892:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773551.273897:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534702336:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773551.273903:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534702464:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773551.273920:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773551.273931:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773551/real 1790773551] req@ffff9c4ec7a7e300 x1877759534702464/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773551.273943:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534702464/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773551.273954:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773551.273958:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773551.273963:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534702464:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773551.273969:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534702592:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773551.273986:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773551.273996:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773551/real 1790773551] req@ffff9c4ec7a7c700 x1877759534702592/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773551.274009:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534702592/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773551.274019:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773551.274023:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773551.274027:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534702592:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773551.336759:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773551.336759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773551.336778:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773551.336791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773551.400721:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773551.400734:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773551.400848:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773551.528722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773551.528722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773551.592711:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773551.656770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773551.720741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773551.720755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773551.720756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773551.784747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773551.848740:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773551.848741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773551.848747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773551.976743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773551.976744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773551.976780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773551.976781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773552.040714:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773552.040765:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773552.040787:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773552.040802:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773552.040803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773552.040839:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773552.168750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773552.232777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773552.296723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773552.360726:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773552.360735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773552.360766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773552.360766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773552.424737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773552.424796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773552.424847:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773552.552767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773552.552772:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773552.616719:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773552.680722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773552.744758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773552.744759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773552.744769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773552.808781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773552.872750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773552.872750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773552.872782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773553.000738:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773553.000772:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773553.000772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773553.000776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773553.064730:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773553.064741:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773553.064747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773553.064766:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773553.064785:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773553.064788:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773553.192710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773553.256736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773553.320718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773553.384764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773553.384764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773553.384787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773553.384789:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773553.448823:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773553.448844:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773553.448863:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773553.576710:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773553.576789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773553.640733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773553.704742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773553.768750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773553.768750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773553.768768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773553.832783:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773553.896727:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773553.896727:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773553.896759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773554.024729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773554.024729:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773554.024765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773554.024765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773554.088770:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773554.088777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773554.088788:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773554.088789:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773554.088839:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773554.088853:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773554.216723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773554.280807:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773554.344712:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773554.408723:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773554.408780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773554.408787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773554.408788:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773554.472778:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773554.472851:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773554.472862:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773554.600716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773554.600811:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773554.664722:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773554.728743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773554.792714:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773554.792738:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773554.792739:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773554.856742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773554.920748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773554.920748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773554.920748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773555.048754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773555.048755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773555.048755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773555.048755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773555.112719:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773555.112724:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773555.112747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773555.112766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773555.112771:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773555.112790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773555.240772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773555.304760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773555.368737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773555.432764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773555.432764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773555.432764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773555.432764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773555.496737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773555.496770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773555.496785:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773555.624749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773555.624749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773555.688795:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773555.752740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773555.816748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773555.816748:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773555.816748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773555.880732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773555.944724:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773555.944732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773555.944752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773556.072733:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773556.072736:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773556.072737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773556.072746:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773556.136734:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773556.136734:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773556.136753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773556.136783:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773556.136784:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773556.136790:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773556.264739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773556.328735:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773556.392725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773556.392771:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773556.392780:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773556.392784:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773556.392795:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2552 00000100:00080000:0.0:1790773556.392807:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773556.392821:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773556.392876:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534702720/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773556.392906:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773556.392909:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773556.392911:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773556.392915:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2552 00000100:00080000:0.0:1790773556.392919:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773556.392925:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773556.392936:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773556.392947:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534702720:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773556.392951:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534702848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773556.392968:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773556.392971:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773556.392974:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773556.392979:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2552 00000100:00080000:0.0:1790773556.392981:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773556.392985:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773556.392989:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773556.392997:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534702976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773556.393006:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773556.393009:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773556.393011:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773556.393014:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2552 00000100:00080000:0.0:1790773556.393021:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773556.393024:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773556.393028:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:1.0:1790773556.393034:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773556.393040:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534703104/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773556.393048:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773556.393053:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773556.393055:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773556.393060:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2552 00000100:00080000:0.0:1790773556.393063:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773556.393068:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773556.393072:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773556.393085:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534703232/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773556.393086:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773556/real 1790773556] req@ffff9c4ec7a7d180 x1877759534702720/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773556.393107:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534702720/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773556.393112:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773556.393121:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773556.393121:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773556.393135:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773556.393135:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773556.393148:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534702720:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773556.393153:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2552 00000100:00080000:0.0:1790773556.393162:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773556.393171:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773556.393177:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534702848:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773556.393178:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773556.393207:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534703360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773556.393214:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773556.393249:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773556.393254:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773556/real 1790773556] req@ffff9c4ec7a7c700 x1877759534702848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773556.393262:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773556.393277:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773556.393286:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2552 00000100:00080000:0.0:1790773556.393297:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773556.393303:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534702848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773556.393308:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773556.393322:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773556.393342:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773556.393344:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534703488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773556.393359:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773556.393373:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773556.393377:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534702848:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773556.393381:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773556.393387:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773556.393393:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534702976:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773556.393394:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2552 00000100:00080000:0.0:1790773556.393404:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773556.393411:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773556.393415:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773556.393428:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534703616/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773556.393435:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773556.393439:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773556.393444:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773556.393447:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773556.393452:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2552 00000100:00080000:0.0:1790773556.393459:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773556.393464:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773556.393471:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773556.393485:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534703744/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773556.393487:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773556/real 1790773556] req@ffff9c4ec7a7e300 x1877759534702976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773556.393530:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534702976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773556.393540:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773556.393545:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773556.393548:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534702976:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773556.393555:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534703104:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773556.393578:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773556.393588:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773556/real 1790773556] req@ffff9c4ec7a7fb80 x1877759534703104/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773556.393600:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534703104/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773556.393609:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773556.393612:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773556.393616:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534703104:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773556.393622:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534703232:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773556.393635:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773556.393644:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773556/real 1790773556] req@ffff9c4ec7a7ca80 x1877759534703232/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773556.393655:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534703232/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773556.393663:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773556.393691:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773556.393694:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534703232:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773556.393714:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534703360:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773556.393732:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773556.393742:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773556/real 1790773556] req@ffff9c4ec7a7c380 x1877759534703360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773556.393753:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534703360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773556.393767:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773556.393771:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773556.393774:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534703360:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773556.393779:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534703488:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773556.393793:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773556.393801:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773556/real 1790773556] req@ffff9c4ec7a7d880 x1877759534703488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773556.393811:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534703488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773556.393819:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773556.393822:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773556.393825:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534703488:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773556.393830:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534703616:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773556.393843:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773556.393856:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773556/real 1790773556] req@ffff9c4ec7a7ed80 x1877759534703616/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773556.393872:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534703616/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773556.393886:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773556.393891:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773556.393896:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534703616:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773556.393905:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534703744:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773556.393931:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773556.393942:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773556/real 1790773556] req@ffff9c4ec7a7df80 x1877759534703744/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773556.393953:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534703744/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773556.393962:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773556.393965:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773556.393969:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534703744:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773556.456753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773556.456757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773556.456758:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773556.456772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773556.520726:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773556.520783:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773556.520819:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773556.648722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773556.648722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773556.712714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773556.776732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773556.840747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773556.840748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773556.840749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773556.904720:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773556.968740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773556.968740:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773556.968746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773557.096723:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773557.096728:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773557.096736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773557.096749:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773557.160724:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773557.160734:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773557.160734:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773557.160738:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773557.160762:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773557.160762:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773557.288718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773557.352721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773557.416708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773557.480764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773557.480764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773557.480764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773557.480764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773557.544745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773557.544754:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773557.544756:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773557.672735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773557.672736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773557.736742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773557.800765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773557.864703:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773557.864719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773557.864725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773557.928757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773557.992720:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773557.992721:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773557.992774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773558.120725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773558.120729:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773558.120730:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773558.120761:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773558.184740:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773558.184770:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773558.184770:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773558.184771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773558.184809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773558.184812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773558.312744:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773558.376772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773558.440706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773558.504732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773558.504760:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773558.504760:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773558.504761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773558.568725:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773558.568788:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773558.568837:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773558.696711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773558.696742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773558.760714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773558.824778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773558.888697:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773558.888702:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773558.888757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773558.952777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773559.016711:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773559.016726:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773559.016728:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773559.144716:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773559.144721:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773559.144756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773559.144778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773559.208726:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773559.208755:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773559.208772:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773559.208778:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773559.208779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773559.208788:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773559.336710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773559.400733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773559.464705:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773559.528751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773559.528751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773559.528752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773559.528763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773559.592740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773559.592761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773559.592817:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773559.720731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773559.720763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773559.784752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773559.848781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773559.912713:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773559.912725:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773559.912727:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773559.976721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773560.040778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773560.040778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773560.040785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773560.168750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773560.168756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773560.168764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773560.168769:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773560.232744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773560.232752:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773560.232754:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773560.232775:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773560.232801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773560.232817:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773560.360716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773560.424739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773560.488743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773560.552768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773560.552768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773560.552787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773560.552787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773560.616758:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773560.616758:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773560.616809:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773560.744726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773560.744765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773560.808738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773560.872781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773560.936748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773560.936749:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773560.936749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773561.000742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773561.064739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773561.064747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773561.064775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773561.192780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773561.192787:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773561.192804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773561.192818:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773561.256733:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773561.256737:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773561.256751:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773561.256772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773561.256792:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773561.256809:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773561.384712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773561.448733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773561.512734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773561.512737:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773561.512751:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773561.512756:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773561.512768:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2557 00000100:00080000:0.0:1790773561.512780:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773561.512792:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773561.512831:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534703872/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773561.512862:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773561.512867:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773561.512869:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773561.512874:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2557 00000100:00080000:0.0:1790773561.512880:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773561.512887:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773561.512900:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773561.512904:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534703872:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773561.512924:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534704000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773561.512940:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773561.512945:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773561.512948:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773561.512954:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2557 00000100:00080000:0.0:1790773561.512977:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773561.512984:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773561.512993:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773561.512995:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773561.513014:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534704128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773561.513027:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773561.513031:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773561.513034:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773561.513042:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2557 00000100:00080000:0.0:1790773561.513048:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773561.513053:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773561/real 1790773561] req@ffff9c4ec7a7f480 x1877759534703872/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773561.513060:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773561.513073:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773561.513078:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534703872/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773561.513091:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773561.513096:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534704256/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773561.513102:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773561.513132:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534703872:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773561.513156:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773561.513161:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773561.513163:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773561.513172:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2557 00000100:00100000:1.0:1790773561.513177:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534704000:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773561.513181:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773561.513193:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773561.513199:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:1.0:1790773561.513208:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773561.513220:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534704384/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773561.513236:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773561.513241:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773561/real 1790773561] req@ffff9c4ec7a7df80 x1877759534704000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773561.513243:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773561.513248:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773561.513259:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2557 00000100:00080000:0.0:1790773561.513269:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773561.513281:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773561.513283:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534704000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773561.513290:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773561.513317:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773561.513320:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534704512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773561.513324:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773561.513336:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534704000:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773561.513357:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534704128:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773561.513363:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773561.513369:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773561.513372:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773561.513380:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2557 00000100:00080000:0.0:1790773561.513389:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773561.513397:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773561.513404:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773561.513411:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773561.513417:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773561/real 1790773561] req@ffff9c4ec7a7ed80 x1877759534704128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773561.513430:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534704640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773561.513431:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534704128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773561.513483:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773561.513489:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773561.513489:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773561.513498:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534704128:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773561.513510:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1790773561.513514:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534704256:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773561.513517:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773561.513531:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2557 00000100:00080000:0.0:1790773561.513540:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773561.513544:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773561.513559:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773561.513565:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773561/real 1790773561] req@ffff9c4ec7a7d880 x1877759534704256/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773561.513569:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773561.513581:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534704256/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773561.513592:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773561.513598:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534704768/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773561.513612:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773561.513623:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534704256:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773561.513626:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773561.513632:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773561.513635:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773561.513645:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534704384:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773561.513646:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2557 00000100:00080000:0.0:1790773561.513693:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773561.513695:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773561.513709:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773561.513714:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773561/real 1790773561] req@ffff9c4ec7a7c380 x1877759534704384/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773561.513721:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773561.513726:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534704384/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773561.513739:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773561.513745:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773561.513747:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534704896/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773561.513758:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534704384:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773561.513776:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534704512:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773561.513807:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773561.513817:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773561/real 1790773561] req@ffff9c4ec7a7ca80 x1877759534704512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773561.513826:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534704512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773561.513838:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773561.513841:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773561.513843:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534704512:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773561.513848:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534704640:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773561.513860:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773561.513867:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773561/real 1790773561] req@ffff9c4ec7a7fb80 x1877759534704640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773561.513875:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534704640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773561.513881:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773561.513883:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773561.513885:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534704640:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773561.513890:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534704768:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773561.513901:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773561.513907:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773561/real 1790773561] req@ffff9c4ec7a7e300 x1877759534704768/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773561.513915:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534704768/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773561.513922:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773561.513924:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773561.513926:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534704768:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773561.513938:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534704896:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773561.513952:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773561.513962:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773561/real 1790773561] req@ffff9c4ec7a7c700 x1877759534704896/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773561.513975:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534704896/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773561.513986:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773561.513990:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773561.513994:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534704896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773561.576811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773561.576812:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773561.576817:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773561.576817:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773561.640748:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773561.640787:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773561.640846:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773561.768738:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773561.768794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773561.832730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773561.896786:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773561.960694:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773561.960745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773561.960752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773562.024714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773562.088780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773562.088780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773562.088788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773562.216786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773562.216793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773562.216794:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773562.216795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773562.280731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773562.280775:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773562.280777:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773562.280777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773562.280794:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773562.280806:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773562.408735:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773562.472737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773562.536808:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773562.600815:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773562.600815:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773562.600821:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773562.600835:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773562.664747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773562.664799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773562.664907:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773562.792739:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773562.792799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773562.856753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773562.920811:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773562.984723:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773562.984846:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773562.984862:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773563.048730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773563.112740:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773563.112741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773563.112759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773563.240765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773563.240765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773563.240781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773563.240790:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773563.304755:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773563.304836:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773563.304837:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773563.304857:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773563.304860:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773563.304913:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773563.432790:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773563.496825:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773563.560746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773563.624747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773563.624747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773563.624800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773563.624814:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773563.688767:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773563.688839:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773563.688890:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773563.816716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773563.816755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773563.880764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773563.944744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773564.008787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773564.008787:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773564.008800:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773564.072730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773564.136786:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773564.136786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773564.136843:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773564.264729:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773564.264738:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773564.264754:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773564.264766:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773564.328733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773564.328744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773564.328745:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773564.328779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773564.328801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773564.328812:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773564.456761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773564.520734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773564.584737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773564.648789:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773564.648789:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773564.648799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773564.648822:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773564.712787:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773564.712831:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773564.712952:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773564.840754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773564.840761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773564.904755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773564.968725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773565.032769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773565.032770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773565.032770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773565.096727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773565.160743:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773565.160743:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773565.160746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773565.288780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773565.288783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773565.288808:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773565.288813:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773565.352741:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773565.352746:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773565.352752:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773565.352762:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773565.352806:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773565.352815:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773565.480765:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773565.544743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773565.608725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773565.672791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773565.672793:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773565.672794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773565.672804:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773565.736783:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773565.736912:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773565.736937:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773565.864740:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773565.864796:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773565.928792:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773565.992821:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773566.056790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773566.056842:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773566.056843:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773566.120804:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773566.184797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773566.184810:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773566.184825:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773566.312737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773566.312758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773566.312770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773566.312771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773566.376732:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773566.376771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773566.376798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773566.376803:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773566.376831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773566.376836:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773566.504754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773566.568761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773566.632706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773566.632742:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773566.632749:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773566.632753:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773566.632761:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2562 00000100:00080000:3.0:1790773566.632770:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773566.632779:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773566.632820:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534705024/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773566.632842:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773566.632845:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773566.632847:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773566.632850:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2562 00000100:00080000:3.0:1790773566.632856:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773566.632860:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773566.632868:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773566.632879:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534705152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773566.632890:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534705024:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773566.632906:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773566.632911:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773566.632914:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773566.632922:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2562 00000100:00080000:3.0:1790773566.632927:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773566.632933:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773566.632940:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773566.632954:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534705280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773566.632963:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773566.632966:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773566.632967:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773566.632971:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2562 00000100:00080000:3.0:1790773566.632976:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773566.632983:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773566.632990:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773566.632996:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773566.633009:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534705408/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773566.633016:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773566.633032:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773566.633034:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773566.633040:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2562 00000100:00080000:3.0:1790773566.633044:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773566.633051:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773566.633056:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773566.633061:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773566/real 1790773566] req@ffff9c4ef73c1c00 x1877759534705024/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773566.633072:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534705536/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773566.633083:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773566.633086:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1790773566.633090:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534705024/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773566.633090:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773566.633101:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2562 00000100:00080000:3.0:1790773566.633104:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773566.633107:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773566.633110:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773566.633118:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534705664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773566.633175:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773566.633176:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773566.633188:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773566.633189:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773566.633196:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773566.633203:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534705024:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773566.633210:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2562 00000100:00080000:3.0:1790773566.633218:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773566.633226:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773566.633232:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773566.633243:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534705152:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773566.633253:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534705792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773566.633269:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773566.633276:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773566.633277:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773566.633280:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773566.633287:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2562 00000100:00080000:3.0:1790773566.633294:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773566.633296:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773566/real 1790773566] req@ffff9c4ef73c2300 x1877759534705152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773566.633304:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773566.633319:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773566.633342:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534705920/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773566.633343:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534705152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773566.633356:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773566.633369:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773566.633376:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773566.633390:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2562 00000100:00080000:3.0:1790773566.633412:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773566.633419:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773566.633420:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773566.633433:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:3.0:1790773566.633435:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773566.633442:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534705152:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773566.633453:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534706048/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773566.633456:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534705280:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773566.633510:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773566.633524:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773566/real 1790773566] req@ffff9c4ef73c0e00 x1877759534705280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773566.633535:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534705280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773566.633542:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773566.633545:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773566.633547:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534705280:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773566.633551:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534705408:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773566.633571:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773566.633579:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773566/real 1790773566] req@ffff9c4ef73c0a80 x1877759534705408/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773566.633587:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534705408/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773566.633599:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773566.633602:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773566.633604:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534705408:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773566.633610:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534705536:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773566.633621:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773566.633627:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773566/real 1790773566] req@ffff9c4ef73c3800 x1877759534705536/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773566.633635:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534705536/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773566.633642:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773566.633645:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773566.633647:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534705536:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773566.633671:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534705664:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773566.633685:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773566.633693:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773566/real 1790773566] req@ffff9c4ef73c2d80 x1877759534705664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773566.633701:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534705664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773566.633715:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773566.633718:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773566.633723:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534705664:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773566.633746:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534705792:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773566.633780:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773566.633791:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773566/real 1790773566] req@ffff9c4ef73c2a00 x1877759534705792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773566.633800:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534705792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773566.633806:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773566.633809:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773566.633811:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534705792:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773566.633817:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534705920:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773566.633847:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773566.633859:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773566/real 1790773566] req@ffff9c4ef73c0700 x1877759534705920/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773566.633873:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534705920/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773566.633885:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773566.633888:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773566.633891:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534705920:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773566.633906:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534706048:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773566.633921:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773566.633930:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773566/real 1790773566] req@ffff9c4ef73c1500 x1877759534706048/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773566.633939:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534706048/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773566.633946:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773566.633949:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773566.633951:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534706048:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773566.696767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773566.696800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773566.696825:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773566.696827:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773566.760774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773566.760813:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773566.760923:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773566.888757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773566.888765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773566.952728:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773567.016728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773567.080806:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773567.080806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773567.080814:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773567.144796:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773567.208747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773567.208747:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773567.208780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773567.336750:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773567.336750:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773567.336750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773567.336802:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773567.400733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773567.400734:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773567.400759:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773567.400773:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773567.400792:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773567.400818:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773567.528777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773567.592784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773567.656764:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773567.720727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773567.720727:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773567.720730:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773567.720805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773567.784758:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773567.784760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773567.784844:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773567.912748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773567.912748:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773567.976754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773568.040773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773568.104723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773568.104735:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773568.104764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773568.168724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773568.232776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773568.232776:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773568.232776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773568.360755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773568.360759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773568.360766:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773568.360767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773568.424722:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773568.424730:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773568.424748:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773568.424766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773568.424787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773568.424798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773568.552722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773568.616770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773568.680734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773568.744702:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773568.744755:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773568.744780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773568.744784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773568.808759:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773568.808774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773568.808826:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773568.936736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773568.936741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773569.000730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773569.064756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773569.128724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773569.128739:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773569.128740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773569.192716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773569.256736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773569.256736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773569.256736:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773569.384731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773569.384735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773569.384735:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773569.384741:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773569.448731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773569.448734:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773569.448740:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773569.448743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773569.448766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773569.448775:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773569.576723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773569.640734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773569.704776:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773569.768735:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773569.768746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773569.768748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773569.768756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773569.832747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773569.832757:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773569.832893:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773569.960747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773569.960747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773570.024749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773570.088751:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773570.152730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773570.152730:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773570.152734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773570.216728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773570.280764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773570.280764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773570.280765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773570.408756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773570.408756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773570.408756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773570.408768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773570.472731:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773570.472732:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773570.472732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773570.472741:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773570.472769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773570.472784:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773570.600720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773570.664742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773570.728772:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773570.792775:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773570.792775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773570.792775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773570.792776:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773570.856761:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773570.856784:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773570.856786:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773570.984714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773570.984737:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773571.048724:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773571.112749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773571.176769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773571.176769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773571.176770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773571.240739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773571.304734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773571.304735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773571.304737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773571.432761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773571.432761:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773571.432761:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773571.432761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773571.496730:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773571.496746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773571.496754:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773571.496762:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773571.496769:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773571.496784:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773571.624729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773571.688760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773571.752740:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773571.752749:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773571.752753:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773571.752763:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2567 00000100:00100000:0.0:1790773571.752770:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773571.752773:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773571.752787:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773571.752820:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534706176/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773571.752842:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773571.752845:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773571.752846:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773571.752850:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2567 00000100:00080000:3.0:1790773571.752852:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773571.752857:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773571.752865:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773571.752879:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534706176:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773571.752879:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534706304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773571.752889:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773571.752893:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773571.752894:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773571.752898:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2567 00000100:00080000:3.0:1790773571.752900:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773571.752903:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773571.752907:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773571.752916:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534706432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773571.752946:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773571.752948:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773571.752950:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773571.752953:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2567 00000100:00080000:3.0:1790773571.752957:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773571.752961:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773571.752966:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:1.0:1790773571.752967:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773571.752981:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534706560/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773571.752989:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773571.752991:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773571.752992:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773571.752997:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2567 00000100:00080000:3.0:1790773571.753000:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773571.753005:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773571.753009:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773571.753010:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773571/real 1790773571] req@ffff9c4ef73c0000 x1877759534706176/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773571.753021:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534706688/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773571.753030:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773571.753031:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534706176/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773571.753036:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773571.753039:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773571.753047:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2567 00000100:00080000:1.0:1790773571.753051:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773571.753051:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773571.753058:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773571.753059:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:3.0:1790773571.753065:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773571.753067:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534706176:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773571.753076:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534706816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773571.753083:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773571.753085:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773571.753087:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773571.753091:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2567 00000100:00100000:1.0:1790773571.753095:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534706304:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773571.753095:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773571.753103:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773571.753107:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773571.753117:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534706944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773571.753124:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773571.753134:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773571.753137:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773571.753138:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773571.753142:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773571/real 1790773571] req@ffff9c4ef73c1500 x1877759534706304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773571.753145:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2567 00000100:00080000:3.0:1790773571.753151:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773571.753157:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773571.753161:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773571.753169:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534706304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773571.753171:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534707072/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773571.753196:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773571.753205:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773571.753209:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773571.753212:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2567 00000100:00080000:3.0:1790773571.753215:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773571.753220:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773571.753221:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773571.753223:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773571.753229:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790773571.753233:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534707200/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773571.753235:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534706304:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773571.753246:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534706432:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773571.753264:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773571.753272:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773571/real 1790773571] req@ffff9c4ef73c0700 x1877759534706432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773571.753281:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534706432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773571.753288:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773571.753291:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773571.753293:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534706432:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773571.753297:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534706560:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773571.753308:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773571.753314:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773571/real 1790773571] req@ffff9c4ef73c2a00 x1877759534706560/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773571.753322:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534706560/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773571.753329:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773571.753331:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773571.753335:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534706560:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773571.753339:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534706688:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773571.753351:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773571.753361:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773571/real 1790773571] req@ffff9c4ef73c2d80 x1877759534706688/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773571.753374:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534706688/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773571.753384:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773571.753388:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773571.753393:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534706688:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773571.753400:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534706816:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773571.753420:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773571.753429:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773571/real 1790773571] req@ffff9c4ef73c3800 x1877759534706816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773571.753438:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534706816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773571.753451:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773571.753453:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773571.753456:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534706816:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773571.753472:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534706944:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773571.753484:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773571.753491:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773571/real 1790773571] req@ffff9c4ef73c0a80 x1877759534706944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773571.753500:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534706944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773571.753506:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773571.753508:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773571.753510:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534706944:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773571.753514:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534707072:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773571.753525:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773571.753531:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773571/real 1790773571] req@ffff9c4ef73c0e00 x1877759534707072/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773571.753539:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534707072/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773571.753547:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773571.753549:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773571.753551:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534707072:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773571.753563:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534707200:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773571.753575:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773571.753582:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773571/real 1790773571] req@ffff9c4ef73c2300 x1877759534707200/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773571.753591:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534707200/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773571.753599:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773571.753601:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773571.753604:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534707200:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773571.816751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773571.816751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773571.816751:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773571.816809:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773571.880795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773571.880858:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773571.881000:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773572.008745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773572.008745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773572.072725:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773572.136736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773572.200713:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773572.200736:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773572.200745:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773572.264725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773572.328720:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773572.328720:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773572.328721:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773572.456725:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773572.456725:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773572.456732:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773572.456733:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773572.520714:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773572.520715:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773572.520732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773572.520763:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773572.520821:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773572.520825:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773572.648723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773572.712721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773572.776708:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773572.840755:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773572.840755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773572.840755:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773572.840756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773572.904733:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773572.904755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773572.904852:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773573.032746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773573.032747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773573.096704:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773573.160750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773573.224727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773573.224727:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773573.224727:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773573.288749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773573.352737:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773573.352737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773573.352741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773573.480761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773573.480762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773573.480762:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773573.480763:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773573.544745:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773573.544746:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773573.544747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773573.544769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773573.544780:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773573.544792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773573.672720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773573.736745:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773573.800726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773573.864705:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773573.864709:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773573.864742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773573.864745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773573.928776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773573.928808:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773573.928813:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773574.056722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773574.056724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773574.120741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773574.184734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773574.248722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773574.248722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773574.248743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773574.312730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773574.376760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773574.376760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773574.376760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773574.504715:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773574.504715:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773574.504717:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773574.504732:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773574.568704:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773574.568718:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773574.568724:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773574.568738:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773574.568747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773574.568754:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773574.696716:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773574.760743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773574.824729:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773574.888762:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773574.888800:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773574.888800:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773574.888802:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773574.952746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773574.952810:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773574.952814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773575.080782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773575.080791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773575.144739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773575.208740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773575.272732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773575.272741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773575.272774:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773575.336740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773575.400802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773575.400803:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773575.400803:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773575.528755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773575.528755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773575.528758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773575.528788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773575.592740:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773575.592763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773575.592764:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773575.592765:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773575.592803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773575.592807:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773575.720710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773575.784741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773575.848764:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773575.912809:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773575.912829:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773575.912830:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773575.912891:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773575.976805:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773575.976925:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773575.977051:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773576.104754:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773576.104786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773576.168766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773576.232792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773576.296746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773576.296773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773576.296806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773576.360761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773576.424770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773576.424776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773576.424793:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773576.552783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773576.552816:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773576.552823:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773576.552823:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773576.616733:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773576.616762:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773576.616808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773576.616869:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773576.616889:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773576.616908:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773576.744706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773576.808794:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773576.872782:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773576.872870:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773576.872880:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773576.872886:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773576.872898:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2572 00000100:00080000:2.0:1790773576.872912:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773576.872926:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773576.872985:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534707328/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773576.873026:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773576.873032:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773576.873035:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773576.873040:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2572 00000100:00080000:2.0:1790773576.873045:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773576.873051:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773576.873065:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773576.873092:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534707456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773576.873121:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773576.873124:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534707328:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773576.873139:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773576.873147:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773576.873158:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2572 00000100:00080000:2.0:1790773576.873165:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773576.873180:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773576.873190:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773576.873215:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534707584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773576.873229:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773576.873234:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773576.873237:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773576.873248:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773576.873251:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2572 00000100:00080000:2.0:1790773576.873258:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773576.873263:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773576.873272:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773576.873292:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534707712/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773576.873304:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773576.873310:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773576.873313:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773576.873320:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2572 00000100:00080000:2.0:1790773576.873326:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773576.873333:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773576/real 1790773576] req@ffff9c4efbcc0a80 x1877759534707328/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773576.873339:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773576.873352:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773576.873377:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534707840/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773576.873387:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534707328/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773576.873403:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773576.873425:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773576.873431:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773576.873432:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773576.873443:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2572 00000100:00080000:1.0:1790773576.873445:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773576.873453:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773576.873457:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534707328:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773576.873467:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773576.873474:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773576.873489:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534707968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773576.873499:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534707456:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773576.873516:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773576.873523:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773576.873526:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773576.873534:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2572 00000100:00080000:2.0:1790773576.873540:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773576.873548:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773576.873554:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773576.873564:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773576.873576:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773576/real 1790773576] req@ffff9c4efbcc2a00 x1877759534707456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773576.873587:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534708096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773576.873610:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534707456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773576.873612:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773576.873631:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773576.873638:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773576.873690:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2572 00000100:00080000:2.0:1790773576.873696:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773576.873707:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773576.873720:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773576.873744:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534708224/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773576.873752:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773576.873759:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773576.873768:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773576.873769:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773576.873777:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773576.873780:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534707456:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773576.873791:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2572 00000100:00080000:2.0:1790773576.873798:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773576.873803:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534707584:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773576.873812:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773576.873847:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773576.873869:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534708352/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773576.873897:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773576.873921:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773576/real 1790773576] req@ffff9c4efbcc2300 x1877759534707584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773576.873940:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534707584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773576.873952:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773576.873957:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773576.873962:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534707584:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773576.873971:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534707712:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773576.873993:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773576.874006:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773576/real 1790773576] req@ffff9c4efbcc1880 x1877759534707712/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773576.874021:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534707712/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773576.874034:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773576.874038:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773576.874043:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534707712:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773576.874051:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534707840:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773576.874072:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773576.874091:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773576/real 1790773576] req@ffff9c4efbcc1500 x1877759534707840/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773576.874116:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534707840/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773576.874148:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773576.874156:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773576.874164:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534707840:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773576.874204:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534707968:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773576.874237:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773576.874254:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773576/real 1790773576] req@ffff9c4efbcc2680 x1877759534707968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773576.874270:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534707968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773576.874290:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773576.874295:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773576.874300:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534707968:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773576.874308:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534708096:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773576.874339:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773576.874352:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773576/real 1790773576] req@ffff9c4efbcc0700 x1877759534708096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773576.874367:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534708096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773576.874378:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773576.874381:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773576.874386:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534708096:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773576.874393:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534708224:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773576.874420:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773576.874432:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773576/real 1790773576] req@ffff9c4efbcc1f80 x1877759534708224/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773576.874447:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534708224/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773576.874459:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773576.874463:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773576.874467:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534708224:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773576.874474:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534708352:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773576.874493:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773576.874513:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773576/real 1790773576] req@ffff9c4efbcc1c00 x1877759534708352/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773576.874527:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534708352/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773576.874540:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773576.874545:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773576.874549:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534708352:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773576.936777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773576.936786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773576.936813:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773576.936814:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773577.000787:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773577.000892:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773577.001020:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773577.128742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773577.128784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773577.192849:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773577.256763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773577.320712:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773577.320721:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773577.320723:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773577.384776:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773577.448756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773577.448756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773577.448762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773577.576713:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773577.576714:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773577.576737:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773577.576737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773577.640710:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773577.640721:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773577.640728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773577.640734:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773577.640747:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773577.640753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773577.768753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773577.832738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773577.896714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773577.960783:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773577.960783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773577.960844:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773577.960852:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773578.024728:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773578.024733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773578.024776:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773578.152705:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773578.152729:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773578.216746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773578.280735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773578.344758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773578.344758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773578.344758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773578.408719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773578.472712:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773578.472736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773578.472736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773578.600727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773578.600727:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773578.600728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773578.600732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773578.664701:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773578.664711:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773578.664728:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773578.664729:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773578.664745:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773578.664748:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773578.792719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773578.856704:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773578.920698:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773578.984730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773578.984730:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773578.984732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773578.984735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773579.048724:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773579.048735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773579.048810:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773579.176710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773579.176711:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773579.240738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773579.304722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773579.368724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773579.368724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773579.368729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773579.432738:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773579.496750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773579.496750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773579.496786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773579.624719:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773579.624721:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773579.624731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773579.624755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773579.688741:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773579.688741:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773579.688741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773579.688748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773579.688787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773579.688788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773579.816705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773579.880752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773579.944721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773580.008776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773580.008776:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773580.008777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773580.008788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773580.072732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773580.072745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773580.072784:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773580.200724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773580.200736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773580.264734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773580.328773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773580.392778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773580.392778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773580.392779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773580.456815:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773580.520790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773580.520815:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773580.520841:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773580.648787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773580.648787:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773580.648804:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773580.648822:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773580.712777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773580.712780:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773580.712783:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773580.712791:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773580.712862:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773580.712862:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773580.840714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773580.904799:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773580.968717:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773581.032781:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773581.032781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773581.032781:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773581.032817:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773581.096857:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773581.096889:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773581.097053:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773581.224732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773581.224757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773581.288798:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773581.352767:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773581.416718:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773581.416719:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773581.416719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773581.480780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773581.544758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773581.544759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773581.544771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773581.672756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773581.672756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773581.672756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773581.672767:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773581.736738:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773581.736747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773581.736761:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773581.736776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773581.736797:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773581.736813:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773581.864738:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773581.928745:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773581.992777:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773581.992780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773581.992789:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773581.992793:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773581.992802:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2577 00000100:00080000:3.0:1790773581.992811:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773581.992821:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773581.992856:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534708480/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773581.992878:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773581.992881:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773581.992883:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773581.992887:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2577 00000100:00080000:3.0:1790773581.992891:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773581.992898:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773581.992909:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773581.992926:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534708480:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773581.992932:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534708608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773581.992952:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773581.992955:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773581.992957:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773581.992963:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2577 00000100:00080000:3.0:1790773581.992971:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773581.992979:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773581.992986:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773581.993003:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534708736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773581.993017:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773581.993047:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773581.993051:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773581.993053:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773581.993061:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2577 00000100:00080000:3.0:1790773581.993068:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773581.993078:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773581.993087:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773581/real 1790773581] req@ffff9c4ef73c1c00 x1877759534708480/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773581.993087:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773581.993115:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534708864/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773581.993116:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534708480/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773581.993157:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773581.993169:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773581.993182:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773581.993187:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534708480:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773581.993204:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773581.993209:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773581.993219:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2577 00000100:00080000:3.0:1790773581.993227:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773581.993231:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534708608:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773581.993240:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773581.993253:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:1.0:1790773581.993260:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773581.993281:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773581/real 1790773581] req@ffff9c4ef73c2300 x1877759534708608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773581.993282:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534708992/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773581.993315:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534708608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773581.993351:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773581.993352:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773581.993365:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773581.993365:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773581.993374:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773581.993377:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534708608:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773581.993384:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2577 00000100:00080000:3.0:1790773581.993390:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773581.993391:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534708736:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773581.993402:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773581.993410:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773581.993417:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773581.993428:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534709120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773581.993435:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773581/real 1790773581] req@ffff9c4ef73c0e00 x1877759534708736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773581.993453:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534708736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773581.993455:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773581.993472:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773581.993478:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773581.993481:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773581.993486:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2577 00000100:00080000:1.0:1790773581.993492:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773581.993494:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773581.993498:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534708736:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773581.993501:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773581.993516:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773581.993536:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534708864:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773581.993542:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534709248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773581.993555:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773581.993560:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773581.993561:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773581.993565:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2577 00000100:00080000:3.0:1790773581.993571:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773581.993579:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773581.993587:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:1.0:1790773581.993610:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773581.993616:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534709376/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773581.993625:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773581.993626:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773581/real 1790773581] req@ffff9c4ef73c0a80 x1877759534708864/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773581.993636:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773581.993642:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773581.993655:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534708864/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773581.993681:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2577 00000100:00080000:3.0:1790773581.993684:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773581.993688:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773581.993690:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773581.993694:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773581.993696:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773581.993709:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534708864:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773581.993714:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534709504/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773581.993751:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534708992:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773581.993773:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773581.993782:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773581/real 1790773581] req@ffff9c4ef73c3800 x1877759534708992/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773581.993791:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534708992/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773581.993809:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773581.993811:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773581.993814:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534708992:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773581.993818:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534709120:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773581.993830:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773581.993837:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773581/real 1790773581] req@ffff9c4ef73c2d80 x1877759534709120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773581.993845:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534709120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773581.993858:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773581.993860:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773581.993862:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534709120:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773581.993866:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534709248:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773581.993877:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773581.993883:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773581/real 1790773581] req@ffff9c4ef73c2a00 x1877759534709248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773581.993919:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534709248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773581.993927:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773581.993931:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773581.993935:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534709248:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773581.993940:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534709376:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773581.993955:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773581.993963:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773581/real 1790773581] req@ffff9c4ef73c0700 x1877759534709376/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773581.993973:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534709376/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773581.993983:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773581.993986:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773581.993989:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534709376:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773581.994023:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534709504:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773581.994035:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773581.994041:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773581/real 1790773581] req@ffff9c4ef73c1500 x1877759534709504/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773581.994049:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534709504/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773581.994056:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773581.994058:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773581.994078:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534709504:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773582.056762:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773582.056762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773582.056786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773582.056797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773582.120769:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773582.120776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773582.120841:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773582.248814:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773582.248815:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773582.312791:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773582.376781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773582.440772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773582.440791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773582.440791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773582.504733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773582.568749:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773582.568763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773582.568763:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773582.696772:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773582.696773:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773582.696783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773582.696803:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773582.760763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773582.760763:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773582.760766:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773582.760779:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773582.760804:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773582.760812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773582.888751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773582.952751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773583.016747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773583.080747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773583.080747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773583.080757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773583.080760:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773583.144777:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773583.144791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773583.144856:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773583.272740:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773583.272745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773583.336767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773583.400757:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773583.464733:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773583.464741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773583.464744:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773583.528718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773583.592767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773583.592767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773583.592767:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773583.720722:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773583.720727:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773583.720735:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773583.720745:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773583.784724:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773583.784727:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773583.784733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773583.784750:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773583.784769:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773583.784772:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773583.912714:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773583.976733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773584.040733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773584.104748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773584.104748:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773584.104763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773584.104782:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773584.168741:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773584.168751:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773584.168800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773584.296735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773584.296738:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773584.360747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773584.424746:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773584.488748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773584.488748:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773584.488748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773584.552762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773584.616710:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773584.616716:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773584.616734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773584.744726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773584.744747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773584.744750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773584.744757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773584.808748:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773584.808750:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773584.808750:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773584.808752:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773584.808783:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773584.808792:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773584.936722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773585.000731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773585.064740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773585.128747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773585.128749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773585.128751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773585.128751:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773585.192739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773585.192758:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773585.192772:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773585.320718:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773585.320727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773585.384729:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773585.448727:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773585.512759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773585.512759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773585.512760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773585.576734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773585.640726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773585.640729:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773585.640748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773585.768742:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773585.768745:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773585.768756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773585.768761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773585.832771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773585.832773:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773585.832775:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773585.832785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773585.832823:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773585.832847:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773585.960710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773586.024754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773586.088764:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773586.152719:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773586.152724:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773586.152789:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773586.152789:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773586.216752:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773586.216768:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773586.216781:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773586.344762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773586.344770:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773586.408749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773586.472740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773586.536766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773586.536766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773586.536775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773586.600762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773586.664755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773586.664755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773586.664780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773586.792777:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773586.792814:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773586.792815:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773586.792815:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773586.856723:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773586.856728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773586.856760:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773586.856772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773586.856786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773586.856786:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773586.984720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773587.048731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773587.112729:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773587.112798:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773587.112809:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773587.112814:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773587.112825:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2583 00000100:00080000:2.0:1790773587.112837:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773587.112850:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773587.112900:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534709632/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773587.112934:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773587.112938:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773587.112941:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773587.112946:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2583 00000100:00080000:2.0:1790773587.112952:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773587.112960:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773587.112970:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773587.112981:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534709632:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773587.112990:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534709760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773587.113007:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773587.113011:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773587.113013:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773587.113019:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2583 00000100:00080000:2.0:1790773587.113023:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773587.113030:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773587.113036:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773587.113052:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534709888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773587.113063:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773587.113067:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773587.113070:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773587.113075:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2583 00000100:00080000:2.0:1790773587.113079:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773587.113086:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773587.113093:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773587.113112:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534710016/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773587.113124:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773587.113128:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773587.113131:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773587.113138:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2583 00000100:00100000:1.0:1790773587.113139:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534709760:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773587.113143:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773587.113156:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773587.113161:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773587.113178:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534710144/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773587.113202:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773587.113207:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773587.113210:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773587.113217:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534709888:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773587.113223:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2583 00000100:00080000:2.0:1790773587.113230:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773587.113238:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773587.113244:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773587.113270:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534710272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773587.113277:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534710016:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773587.113296:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773587.113312:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773587.113314:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773587.113321:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2583 00000100:00080000:2.0:1790773587.113327:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773587.113335:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773587.113342:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773587.113363:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534710144:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773587.113364:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534710400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773587.113394:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773587.113398:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773587.113401:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773587.113406:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2583 00000100:00080000:2.0:1790773587.113411:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773587.113414:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534710272:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773587.113422:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773587.113430:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773587.113447:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534710528/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773587.113460:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773587.113465:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534710400:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773587.113467:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773587.113474:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773587.113483:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2583 00000100:00080000:2.0:1790773587.113488:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773587.113496:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773587.113502:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773587.113515:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534710528:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773587.113518:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534710656/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773587.113569:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534710656:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773587.176740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773587.176740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773587.176760:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773587.176777:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773587.240799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773587.240818:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773587.240821:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773587.368728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773587.368755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773587.432747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773587.496752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773587.560793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773587.560793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773587.560803:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773587.624766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773587.688792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773587.688792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773587.688802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773587.816749:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773587.816749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773587.816749:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773587.816776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773587.880737:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773587.880752:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773587.880756:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773587.880779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773587.880783:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773587.880789:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773588.008718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773588.072718:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773588.136733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773588.200705:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773588.200736:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773588.200765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773588.200768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773588.264770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773588.264778:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773588.264845:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773588.392765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773588.392774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773588.456725:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773588.520723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773588.584739:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773588.584739:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773588.584740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773588.648703:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773588.712732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773588.712732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773588.712738:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773588.840750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773588.840751:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773588.840752:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773588.840752:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773588.904713:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773588.904715:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773588.904722:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773588.904739:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773588.904759:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773588.904766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773589.032699:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773589.096737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773589.160729:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773589.224709:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773589.224714:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773589.224724:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773589.224745:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773589.288738:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773589.288752:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773589.288795:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773589.416710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773589.416718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773589.480696:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773589.544716:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773589.608699:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773589.608713:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773589.608716:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773589.672692:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773589.736718:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773589.736719:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773589.736720:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773589.864711:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773589.864725:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773589.864732:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773589.864763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773589.928739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773589.928740:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773589.928746:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773589.928771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773589.928795:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773589.928803:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773590.056734:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773590.120720:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773590.184706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:2.0:1790773590.184934:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:2.0:1790773590.184970:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1790773590.184984:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773590.185035:0:2384:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:2.0:1790773590.185053:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773590.185066:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773590.185077:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773590.185088:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:1.0:1790773590.185089:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773587/real 1790773590] req@ffff9c4efbcc2300 x1877759534710656/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1790773590.185104:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:1.0:1790773590.185123:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534710656/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1790773590.185128:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773590.185140:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:1.0:1790773590.185157:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773590.185164:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773590.185170:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534710656:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773590.185179:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773587/real 1790773590] req@ffff9c4efbcc1880 x1877759534710528/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773590.185192:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534710528/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773590.185210:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773590.185214:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773590.185217:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534710528:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773590.185225:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773587/real 1790773590] req@ffff9c4efbcc1500 x1877759534710400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773590.185236:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534710400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773590.185243:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773590.185247:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773590.185251:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534710400:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773590.185258:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773587/real 1790773590] req@ffff9c4efbcc0700 x1877759534710144/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773590.185272:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534710144/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773590.185282:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773590.185287:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773590.185291:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534710144:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773590.185299:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773587/real 1790773590] req@ffff9c4efbcc2680 x1877759534710272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773590.185312:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534710272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773590.185330:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773590.185334:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773590.185337:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534710272:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773590.185344:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773587/real 1790773590] req@ffff9c4efbcc3100 x1877759534709760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773590.185356:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534709760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773590.185372:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773590.185376:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773590.185380:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534709760:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773590.185386:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773587/real 1790773590] req@ffff9c4efbcc1c00 x1877759534709888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773590.185398:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534709888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773590.185408:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773590.185412:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773590.185415:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534709888:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773590.185422:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773587/real 1790773590] req@ffff9c4efbcc1f80 x1877759534710016/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773590.185433:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534710016/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773590.185443:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773590.185447:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773590.185450:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534710016:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773590.185457:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773587/real 1790773590] req@ffff9c4efbcc0380 x1877759534709632/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773590.185470:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534709632/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773590.185477:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773590.185481:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773590.185484:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534709632:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773590.248766:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773590.248771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773590.248782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773590.248782:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773590.312773:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773590.312773:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773590.312842:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773590.440727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773590.440733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773590.504714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773590.568704:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773590.632737:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773590.632737:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773590.632771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773590.696715:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773590.760730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773590.760730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773590.760731:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773590.888782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773590.888783:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773590.888783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773590.888859:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773590.952723:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773590.952731:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773590.952743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773590.952755:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773590.952768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773590.952793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773591.080752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773591.144789:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773591.208732:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773591.272736:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773591.272743:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773591.272775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773591.272777:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773591.336762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773591.336810:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773591.336813:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773591.464752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773591.464756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773591.528722:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773591.592802:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773591.656715:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773591.656728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773591.656758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773591.720749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773591.784735:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773591.784763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773591.784767:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773591.912725:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773591.912728:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773591.912750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773591.912758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773591.976741:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773591.976744:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773591.976749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773591.976776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773591.976789:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773591.976813:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773592.104760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773592.168724:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773592.232761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773592.232805:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773592.232809:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773592.232813:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773592.232821:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2588 00000100:00080000:0.0:1790773592.232832:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773592.232843:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773592.232882:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534710784/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773592.232911:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773592.232915:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773592.232916:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773592.232921:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2588 00000100:00080000:0.0:1790773592.232924:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773592.232929:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773592.232938:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773592.232951:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534710912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773592.232961:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773592.232963:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773592.232965:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773592.232968:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2588 00000100:00080000:0.0:1790773592.232971:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773592.232982:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773592.232982:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534710784:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773592.232994:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773592.233010:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534711040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773592.233021:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773592.233025:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773592.233027:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773592.233032:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2588 00000100:00080000:0.0:1790773592.233037:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773592.233043:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773592.233049:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773592.233062:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534711168/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773592.233072:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773592.233076:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773592.233081:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773592.233086:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2588 00000100:00080000:0.0:1790773592.233091:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773592.233104:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773592.233105:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773592.233119:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773592.233138:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534711296/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773592.233149:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773592.233153:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773592.233155:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773592.233160:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2588 00000100:00080000:0.0:1790773592.233164:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773592.233169:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773592.233175:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773592.233192:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534711424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773592.233202:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773592/real 1790773592] req@ffff9c4ec7a7d180 x1877759534710784/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773592.233206:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773592.233219:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773592.233225:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773592.233244:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2588 00000100:00080000:0.0:1790773592.233249:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773592.233258:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534710784/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773592.233259:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773592.233276:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773592.233287:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773592.233296:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534711552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773592.233300:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773592.233310:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534710784:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773592.233333:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773592.233337:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773592.233341:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773592.233347:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2588 00000100:00080000:0.0:1790773592.233351:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773592.233356:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773592.233361:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773592.233374:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534711680/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773592.233384:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534710912:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773592.233392:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773592.233398:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773592.233400:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773592.233408:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2588 00000100:00080000:0.0:1790773592.233417:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773592.233437:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773592.233439:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773592.233448:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773592.233472:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534711808/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773592.233480:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773592/real 1790773592] req@ffff9c4ec7a7c700 x1877759534710912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773592.233515:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534710912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773592.233549:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773592.233555:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773592.233559:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534710912:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773592.233569:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534711040:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773592.233598:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773592.233613:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773592/real 1790773592] req@ffff9c4ec7a7e300 x1877759534711040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773592.233629:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534711040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773592.233646:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773592.233689:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773592.233694:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534711040:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773592.233703:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534711168:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773592.233735:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773592.233749:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773592/real 1790773592] req@ffff9c4ec7a7fb80 x1877759534711168/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773592.233769:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534711168/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773592.233783:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773592.233787:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773592.233792:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534711168:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773592.233801:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534711296:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773592.233826:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773592.233841:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773592/real 1790773592] req@ffff9c4ec7a7ca80 x1877759534711296/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773592.233856:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534711296/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773592.233867:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773592.233871:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773592.233875:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534711296:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773592.233889:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534711424:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773592.233913:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773592.233927:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773592/real 1790773592] req@ffff9c4ec7a7c380 x1877759534711424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773592.233940:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534711424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773592.233961:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773592.233965:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773592.233969:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534711424:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773592.233976:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534711552:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773592.233999:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773592.234012:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773592/real 1790773592] req@ffff9c4ec7a7d880 x1877759534711552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773592.234025:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534711552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773592.234034:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773592.234038:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773592.234041:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534711552:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773592.234071:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534711680:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773592.234094:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773592.234106:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773592/real 1790773592] req@ffff9c4ec7a7ed80 x1877759534711680/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773592.234118:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534711680/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773592.234130:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773592.234133:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773592.234137:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534711680:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773592.234144:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534711808:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773592.234165:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773592.234178:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773592/real 1790773592] req@ffff9c4ec7a7df80 x1877759534711808/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773592.234191:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534711808/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773592.234204:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773592.234210:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773592.234216:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534711808:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773592.296767:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773592.296768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773592.296775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773592.296775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773592.360741:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773592.360748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773592.360865:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773592.488729:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773592.488732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773592.552757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773592.616710:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773592.680722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773592.680722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773592.680723:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773592.744824:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773592.808732:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773592.808732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773592.808764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773592.936732:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773592.936741:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773592.936741:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773592.936743:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773593.000735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773593.000736:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773593.000747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773593.000766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773593.000782:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773593.000795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773593.128762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773593.192759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773593.256736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773593.320717:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773593.320773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773593.320773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773593.320774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773593.384765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773593.384799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773593.384860:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773593.512747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773593.512748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773593.576739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773593.640754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773593.704741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773593.704741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773593.704771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773593.768746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773593.832757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773593.832757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773593.832773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773593.960747:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773593.960747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773593.960753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773593.960754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773594.024740:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773594.024740:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773594.024761:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773594.024788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773594.024806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773594.024835:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773594.152737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773594.216765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773594.280748:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773594.344765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773594.344765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773594.344786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773594.344793:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773594.408758:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773594.408769:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773594.408882:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773594.536707:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773594.536754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773594.600757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773594.664739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773594.728777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773594.728777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773594.728780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773594.792746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773594.856728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773594.856728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773594.856762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773594.984783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773594.984785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773594.984796:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773594.984797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773595.048727:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773595.048731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773595.048739:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773595.048740:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773595.048763:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773595.048780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773595.176719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773595.240771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773595.304737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773595.368747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773595.368747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773595.368747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773595.368753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773595.432750:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773595.432781:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773595.432864:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773595.560722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773595.560725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773595.624735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773595.688747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773595.752724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773595.752724:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773595.752724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773595.816705:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773595.880755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773595.880755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773595.880755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773596.008717:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773596.008727:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773596.008729:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773596.008736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773596.072733:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773596.072735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773596.072766:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773596.072796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773596.072809:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773596.072820:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773596.200768:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773596.264731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773596.328722:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773596.392741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773596.392743:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773596.392762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773596.392797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773596.456762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773596.456801:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773596.456830:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773596.584749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773596.584749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773596.648770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773596.712733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773596.776741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773596.776741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773596.776783:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773596.840762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773596.904721:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773596.904729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773596.904788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773597.032720:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773597.032728:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773597.032728:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773597.032774:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773597.096723:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773597.096742:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773597.096761:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773597.096780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773597.096799:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773597.096805:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773597.224731:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773597.288719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773597.352773:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773597.352774:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773597.352781:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773597.352785:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773597.352796:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2593 00000100:00080000:1.0:1790773597.352806:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773597.352818:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773597.352857:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534711936/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773597.352882:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773597.352886:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773597.352888:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773597.352893:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2593 00000100:00080000:1.0:1790773597.352897:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773597.352903:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773597.352911:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534711936:192.168.204.150@tcp:250:kworker.0 00000008:00100000:1.0:1790773597.352913:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773597.352927:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534712064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773597.352937:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773597.352941:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773597.352943:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773597.352946:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2593 00000100:00080000:1.0:1790773597.352960:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773597.352966:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773597.352971:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773597.352971:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773597.352981:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534712192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773597.352997:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773597.353000:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773597.353002:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773597.353006:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2593 00000100:00080000:1.0:1790773597.353010:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773597.353013:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773597/real 1790773597] req@ffff9c4efac4e300 x1877759534711936/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773597.353015:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773597.353020:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773597.353030:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534712320/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773597.353031:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534711936/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773597.353040:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773597.353042:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773597.353044:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773597.353046:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773597.353046:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773597.353050:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2593 00000100:00100000:0.0:1790773597.353051:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534711936:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773597.353054:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773597.353058:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773597.353062:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773597.353070:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534712448/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773597.353073:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534712064:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773597.353081:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773597.353085:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773597.353087:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773597.353092:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2593 00000800:00000100:0.0:1790773597.353097:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773597.353097:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773597.353103:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773597.353109:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773597.353112:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773597/real 1790773597] req@ffff9c4efac4df80 x1877759534712064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773597.353120:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534712576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773597.353126:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534712064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773597.353130:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773597.353134:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773597.353136:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773597.353140:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2593 00000100:00080000:1.0:1790773597.353143:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773597.353144:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773597.353148:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773597.353148:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773597.353152:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534712064:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790773597.353152:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773597.353158:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534712192:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773597.353162:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534712704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773597.353172:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773597.353175:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773597.353178:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773597.353179:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773597.353182:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2593 00000100:00080000:1.0:1790773597.353186:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773597.353190:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773597/real 1790773597] req@ffff9c4efac4dc00 x1877759534712192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773597.353191:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773597.353196:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773597.353202:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534712192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773597.353216:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534712832/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773597.353219:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773597.353222:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773597.353226:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534712192:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773597.353233:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534712320:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773597.353233:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773597.353237:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773597.353239:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773597.353243:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2593 00000100:00080000:1.0:1790773597.353247:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773597.353252:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773597.353257:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:0.0:1790773597.353266:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773597.353266:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534712960/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773597.353277:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773597/real 1790773597] req@ffff9c4efac4ed80 x1877759534712320/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773597.353289:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534712320/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773597.353300:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773597.353304:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773597.353308:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534712320:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773597.353325:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534712448:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773597.353341:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773597.353349:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773597/real 1790773597] req@ffff9c4efac4c700 x1877759534712448/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773597.353358:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534712448/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773597.353364:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773597.353367:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773597.353370:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534712448:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773597.353374:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534712576:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773597.353385:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773597.353398:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773597/real 1790773597] req@ffff9c4efac4f800 x1877759534712576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773597.353407:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534712576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773597.353419:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773597.353436:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773597.353439:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534712576:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773597.353444:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534712704:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773597.353459:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773597.353467:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773597/real 1790773597] req@ffff9c4efac4ea00 x1877759534712704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773597.353476:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534712704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773597.353481:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773597.353484:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773597.353486:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534712704:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773597.353490:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534712832:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773597.353500:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773597.353507:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773597/real 1790773597] req@ffff9c4efac4e680 x1877759534712832/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773597.353515:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534712832/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773597.353523:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773597.353525:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773597.353527:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534712832:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773597.353530:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534712960:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773597.353541:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773597.353547:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773597/real 1790773597] req@ffff9c4efac4f480 x1877759534712960/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773597.353555:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534712960/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773597.353562:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773597.353564:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773597.353566:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534712960:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773597.416771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773597.416771:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773597.416774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773597.416803:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773597.480727:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773597.480754:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773597.480916:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773597.608703:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773597.608707:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773597.672694:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773597.736718:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773597.800765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773597.800773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773597.800780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773597.864736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773597.928718:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773597.928718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773597.928722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773598.056729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773598.056729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773598.056730:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773598.056735:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773598.120726:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773598.120726:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773598.120735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773598.120740:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773598.120765:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773598.120765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773598.248735:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773598.312736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773598.376739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773598.440740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773598.440740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773598.440741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773598.440741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773598.504719:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773598.504731:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773598.504807:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773598.632714:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773598.632714:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773598.696707:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773598.760716:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773598.824700:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773598.824713:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773598.824718:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773598.888710:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773598.952702:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773598.952706:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773598.952724:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773599.080717:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773599.080722:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773599.080726:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773599.080733:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773599.144725:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773599.144734:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773599.144734:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773599.144743:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773599.144763:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773599.144765:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773599.272713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773599.336754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773599.400718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773599.464740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773599.464754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773599.464765:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773599.464770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773599.528737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773599.528784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773599.528847:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773599.656746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773599.656747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773599.720708:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773599.784755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773599.848741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773599.848763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773599.848764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773599.912726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773599.976728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773599.976736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773599.976743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773600.104730:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773600.104731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773600.104737:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773600.104739:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773600.168716:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773600.168730:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773600.168747:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773600.168761:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773600.168781:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773600.168786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773600.296715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773600.360709:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773600.424703:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773600.488756:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773600.488773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773600.488773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773600.488775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773600.552729:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773600.552768:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773600.552818:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773600.680716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773600.680716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773600.744711:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773600.808725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773600.872726:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773600.872726:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773600.872735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773600.936702:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773601.000725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773601.000726:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773601.000753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773601.128721:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773601.128721:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773601.128786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773601.128790:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773601.192722:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773601.192748:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773601.192771:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773601.192777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773601.192782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773601.192804:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773601.320720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773601.384720:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773601.448703:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773601.512723:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773601.512764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773601.512765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773601.512766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773601.576709:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773601.576714:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773601.576825:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773601.704722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773601.704742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773601.768739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773601.832760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773601.896795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773601.896801:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773601.896801:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773601.960753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773602.024728:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773602.024728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773602.024763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773602.152755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773602.152755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773602.152795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773602.152796:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773602.216741:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773602.216742:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773602.216742:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773602.216744:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773602.216793:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773602.216820:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773602.344718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773602.408703:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773602.472779:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773602.472787:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1790773602.472788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773602.472791:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773602.472808:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2598 00000100:00080000:1.0:1790773602.472823:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773602.472837:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773602.472877:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534713088/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773602.472902:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773602.472905:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773602.472907:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773602.472911:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2598 00000100:00080000:1.0:1790773602.472916:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773602.472924:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773602.472935:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773602.472951:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534713088:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773602.472955:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534713216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773602.472968:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773602.472971:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773602.472972:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773602.472977:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2598 00000100:00080000:1.0:1790773602.472979:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773602.472982:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773602.472986:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773602.472998:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534713344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773602.473005:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773602.473008:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773602.473009:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773602.473012:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2598 00000100:00080000:1.0:1790773602.473015:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773602.473018:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773602.473022:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773602.473029:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534713472/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773602.473037:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773602.473040:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773602.473042:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773602.473045:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2598 00000100:00080000:1.0:1790773602.473048:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773602.473051:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773602.473054:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:0.0:1790773602.473056:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773602.473068:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534713600/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773602.473078:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773602.473082:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773602.473084:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773602.473087:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2598 00000100:00080000:1.0:1790773602.473090:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773602.473093:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773602.473098:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773602.473108:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534713728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773602.473114:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773602.473119:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1790773602.473121:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773602/real 1790773602] req@ffff9c4efac4fb80 x1877759534713088/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773602.473125:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773602.473140:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2598 00000100:00080000:1.0:1790773602.473155:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773602.473164:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773602.473168:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773602.473170:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534713088/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773602.473183:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534713856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773602.473186:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773602.473192:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773602.473198:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773602.473200:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773602.473204:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773602.473209:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534713088:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773602.473214:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2598 00000100:00080000:1.0:1790773602.473223:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773602.473229:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773602.473234:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773602.473243:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534713984/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773602.473249:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534713216:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773602.473267:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773602.473271:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773602.473274:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773602.473279:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2598 00000100:00080000:1.0:1790773602.473282:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773602.473292:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773602.473294:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773602.473302:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773602.473314:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534714112/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773602.473318:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773602/real 1790773602] req@ffff9c4efac4f480 x1877759534713216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773602.473349:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534713216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773602.473364:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773602.473367:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773602.473379:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534713216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773602.473404:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534713344:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773602.473431:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773602.473440:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773602/real 1790773602] req@ffff9c4efac4e680 x1877759534713344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773602.473448:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534713344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773602.473455:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773602.473457:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773602.473460:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534713344:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773602.473465:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534713472:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773602.473477:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773602.473484:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773602/real 1790773602] req@ffff9c4efac4ea00 x1877759534713472/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773602.473491:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534713472/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773602.473498:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773602.473501:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773602.473503:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534713472:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773602.473507:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534713600:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773602.473518:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773602.473525:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773602/real 1790773602] req@ffff9c4efac4f800 x1877759534713600/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773602.473532:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534713600/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773602.473539:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773602.473541:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773602.473543:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534713600:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773602.473547:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534713728:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773602.473558:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773602.473565:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773602/real 1790773602] req@ffff9c4efac4c700 x1877759534713728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773602.473573:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534713728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773602.473584:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773602.473586:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773602.473588:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534713728:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773602.473593:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534713856:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773602.473604:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773602.473610:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773602/real 1790773602] req@ffff9c4efac4ed80 x1877759534713856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773602.473618:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534713856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773602.473624:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773602.473626:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773602.473628:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534713856:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773602.473644:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534713984:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773602.473680:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773602.473688:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773602/real 1790773602] req@ffff9c4efac4dc00 x1877759534713984/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773602.473696:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534713984/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773602.473709:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773602.473711:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773602.473714:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534713984:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773602.473718:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534714112:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773602.473729:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773602.473736:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773602/real 1790773602] req@ffff9c4efac4df80 x1877759534714112/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773602.473744:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534714112/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773602.473751:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773602.473753:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773602.473755:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534714112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773602.536746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773602.536763:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773602.536763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773602.536764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773602.600746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773602.600796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773602.600819:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773602.728749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773602.728755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773602.792774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773602.856741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773602.920759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773602.920782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773602.920793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773602.984751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773603.048796:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773603.048806:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773603.048855:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773603.176749:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773603.176766:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773603.176770:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773603.176771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773603.240759:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773603.240798:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773603.240831:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773603.240840:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773603.240880:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773603.240884:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773603.368763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773603.432761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773603.496762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773603.560724:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773603.560725:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773603.560765:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773603.560789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773603.624831:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773603.624863:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773603.624895:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773603.752742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773603.752821:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773603.816754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773603.880823:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773603.944752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773603.944780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773603.944785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773604.008714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773604.072741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773604.072744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773604.072746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773604.200760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773604.200763:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773604.200788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773604.200802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773604.264789:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773604.264791:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773604.264791:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773604.264791:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773604.264857:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773604.264859:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773604.392736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773604.456743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773604.520748:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773604.584727:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773604.584727:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773604.584731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773604.584763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773604.648747:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773604.648792:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773604.648837:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773604.776703:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773604.776706:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773604.840713:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773604.904848:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773604.968732:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773604.968733:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773604.968735:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773605.032722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773605.096729:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773605.096730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773605.096825:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773605.224711:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773605.224716:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773605.224718:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773605.224722:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773605.288722:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773605.288737:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773605.288748:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773605.288767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773605.288776:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773605.288782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773605.416729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773605.480713:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773605.544753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773605.608716:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773605.608721:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773605.608728:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773605.608728:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773605.672699:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773605.672739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773605.672779:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773605.800732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773605.800732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773605.864736:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773605.928729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773605.992712:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773605.992727:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773605.992750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773606.056706:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773606.120760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773606.120760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773606.120760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773606.248796:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773606.248797:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773606.248797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773606.248798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773606.312708:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773606.312716:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773606.312721:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773606.312754:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773606.312756:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773606.312772:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773606.440726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773606.504724:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773606.568736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773606.632778:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773606.632779:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773606.632779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773606.632794:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773606.696741:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773606.696755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773606.696855:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773606.824745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773606.824754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773606.888779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773606.952846:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773607.016713:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773607.016746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773607.016767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773607.080725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773607.144786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773607.144787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773607.144787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773607.272723:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773607.272782:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773607.272783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773607.272783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773607.336726:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773607.336750:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773607.336757:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773607.336759:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773607.336786:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773607.336793:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773607.464721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773607.528738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773607.592720:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773607.592741:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773607.592749:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773607.592753:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773607.592763:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2603 00000100:00080000:1.0:1790773607.592773:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773607.592785:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773607.592830:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534714240/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773607.592869:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773607.592873:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773607.592875:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773607.592881:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2603 00000100:00080000:1.0:1790773607.592885:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773607.592891:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773607.592899:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534714240:192.168.204.150@tcp:250:kworker.0 00000008:00100000:1.0:1790773607.592901:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773607.592919:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534714368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773607.592931:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773607.592936:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773607.592938:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773607.592943:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2603 00000100:00080000:1.0:1790773607.592955:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773607.592963:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773607.592965:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773607.592971:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773607.592986:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534714496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773607.592997:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773607.593001:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773607.593003:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773607.593008:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2603 00000100:00100000:0.0:1790773607.593013:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773607/real 1790773607] req@ffff9c4efac4e300 x1877759534714240/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773607.593014:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773607.593024:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773607.593032:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773607.593038:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534714240/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773607.593048:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534714624/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773607.593049:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773607.593057:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773607.593065:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534714240:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773607.593073:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773607.593078:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773607.593081:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773607.593087:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2603 00000100:00100000:0.0:1790773607.593091:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534714368:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773607.593093:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773607.593103:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773607.593109:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:0.0:1790773607.593117:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773607.593124:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534714752/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773607.593131:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773607/real 1790773607] req@ffff9c4efac4df80 x1877759534714368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773607.593144:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773607.593149:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534714368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773607.593151:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773607.593156:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773607.593166:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2603 00000100:00080000:1.0:1790773607.593171:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773607.593172:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773607.593178:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773607.593179:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773607.593186:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534714368:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790773607.593188:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773607.593194:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534714496:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773607.593204:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534714880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773607.593211:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773607.593240:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773607.593241:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773607/real 1790773607] req@ffff9c4efac4dc00 x1877759534714496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773607.593251:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773607.593257:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773607.593263:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534714496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773607.593265:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2603 00000100:00080000:1.0:1790773607.593272:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773607.593275:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773607.593281:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773607.593282:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773607.593289:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534714496:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790773607.593291:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773607.593297:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534714624:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773607.593308:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534715008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773607.593320:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773607.593322:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773607.593326:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773607.593329:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773607.593331:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773607/real 1790773607] req@ffff9c4efac4ed80 x1877759534714624/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773607.593337:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2603 00000100:00080000:1.0:1790773607.593344:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773607.593349:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773607.593357:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773607.593368:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534714624/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773607.593372:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534715136/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773607.593405:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773607.593406:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773607.593409:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773607.593413:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773607.593414:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773607.593420:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534714624:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773607.593424:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2603 00000100:00080000:1.0:1790773607.593429:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773607.593436:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773607.593441:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773607.593443:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534714752:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773607.593457:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534715264/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773607.593472:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773607.593489:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773607/real 1790773607] req@ffff9c4efac4c700 x1877759534714752/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773607.593503:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534714752/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773607.593514:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773607.593518:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773607.593521:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534714752:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773607.593526:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534714880:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773607.593543:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773607.593551:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773607/real 1790773607] req@ffff9c4efac4f800 x1877759534714880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773607.593559:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534714880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773607.593576:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773607.593578:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773607.593581:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534714880:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773607.593585:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534715008:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773607.593594:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773607.593601:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773607/real 1790773607] req@ffff9c4efac4ea00 x1877759534715008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773607.593608:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534715008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773607.593614:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773607.593616:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773607.593618:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534715008:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773607.593622:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534715136:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773607.593632:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773607.593638:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773607/real 1790773607] req@ffff9c4efac4e680 x1877759534715136/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773607.593646:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534715136/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773607.593672:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773607.593675:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773607.593678:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534715136:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773607.593690:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534715264:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773607.593704:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773607.593711:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773607/real 1790773607] req@ffff9c4efac4f480 x1877759534715264/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773607.593719:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534715264/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773607.593726:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773607.593728:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773607.593731:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534715264:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773607.656713:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773607.656722:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773607.656731:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773607.656749:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773607.720772:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773607.720821:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773607.720827:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773607.848720:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773607.848746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773607.912731:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773607.976704:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773608.040710:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773608.040761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773608.040761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773608.104726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773608.168749:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773608.168770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773608.168787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773608.296755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773608.296791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773608.296791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773608.296793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773608.360711:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773608.360743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773608.360753:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773608.360766:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773608.360795:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773608.360796:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773608.488754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773608.552761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773608.616737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773608.680729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773608.680729:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773608.680732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773608.680769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773608.744757:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773608.744789:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773608.744878:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773608.872761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773608.872774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773608.936790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773609.000841:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773609.064718:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773609.064742:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773609.065699:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773609.128723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773609.192793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773609.192813:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773609.192828:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773609.320765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773609.320766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773609.320810:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773609.320840:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773609.384741:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773609.384771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773609.384777:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773609.384800:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773609.384810:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773609.384836:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773609.512717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773609.576710:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773609.641699:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773609.704745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773609.704751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773609.704784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773609.704786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773609.768804:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773609.768870:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773609.768918:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773609.896719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773609.896725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773609.960741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773610.024734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773610.088703:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773610.088705:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773610.088726:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773610.152706:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773610.216741:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773610.216741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773610.216742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773610.344763:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773610.344763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773610.344763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773610.344764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773610.408722:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773610.408725:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773610.408737:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773610.408757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773610.408772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773610.408775:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773610.536732:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773610.600743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773610.664828:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773610.728731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773610.728737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773610.728738:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773610.728740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773610.792757:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773610.792778:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773610.792872:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773610.920755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773610.920755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773610.984763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773611.048719:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773611.112700:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773611.112717:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773611.112777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773611.176697:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773611.240735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773611.240735:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773611.240742:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773611.368726:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773611.368739:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773611.368758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773611.368773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773611.432722:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773611.432739:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773611.432760:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773611.432780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773611.432781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773611.432811:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773611.560792:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773611.624717:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773611.688721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773611.752767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773611.752767:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773611.752767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773611.752801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773611.816727:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773611.816770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773611.816860:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773611.944718:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773611.944719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773612.008724:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773612.072731:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773612.136732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773612.136735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773612.136780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773612.200717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773612.264736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773612.264736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773612.264736:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773612.392719:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773612.392721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773612.392752:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773612.392773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773612.456742:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773612.456745:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773612.456745:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773612.456792:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773612.456797:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773612.456803:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773612.584792:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773612.648734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773612.712741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773612.712779:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773612.712790:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773612.712796:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773612.712810:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2608 00000100:00080000:1.0:1790773612.712824:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773612.712840:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773612.712890:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534715392/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773612.712925:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773612.712930:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773612.712933:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773612.712940:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2608 00000100:00080000:1.0:1790773612.712945:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773612.712953:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773612.712963:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534715392:192.168.204.150@tcp:250:kworker.0 00000008:00100000:1.0:1790773612.712967:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773612.712991:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534715520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773612.713013:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773612.713018:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773612.713022:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773612.713028:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2608 00000800:00000100:0.0:1790773612.713034:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773612.713034:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773612.713049:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773612.713056:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773612.713070:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534715648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773612.713083:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773612.713090:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773612/real 1790773612] req@ffff9c4efac4fb80 x1877759534715392/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773612.713092:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773612.713097:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773612.713116:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2608 00000100:00100000:0.0:1790773612.713135:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534715392/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773612.713137:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773612.713149:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773612.713162:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773612.713162:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00080000:0.0:1790773612.713175:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773612.713190:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534715776/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773612.713198:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534715392:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773612.713222:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773612.713226:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773612.713230:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773612.713238:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2608 00000100:00100000:0.0:1790773612.713243:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534715520:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773612.713246:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773612.713257:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773612.713264:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:0.0:1790773612.713271:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773612.713285:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534715904/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773612.713289:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773612/real 1790773612] req@ffff9c4efac4f480 x1877759534715520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773612.713344:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534715520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773612.713346:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773612.713368:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773612.713373:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773612.713380:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773612.713383:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2608 00000100:00080000:0.0:1790773612.713387:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773612.713396:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534715520:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773612.713397:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773612.713422:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534715648:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773612.713423:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773612.713441:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773612.713452:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773612.713464:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534716032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773612.713466:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773612/real 1790773612] req@ffff9c4efac4e680 x1877759534715648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773612.713533:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534715648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773612.713534:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773612.713556:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773612.713564:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773612.713567:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773612.713578:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773612.713579:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2608 00000100:00080000:1.0:1790773612.713589:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773612.713590:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534715648:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773612.713605:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773612.713606:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534715776:192.168.204.150@tcp:8:kworker.0 00000008:00100000:1.0:1790773612.713623:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773612.713636:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773612.713671:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773612/real 1790773612] req@ffff9c4efac4ea00 x1877759534715776/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773612.713681:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534715776/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773612.713689:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534716160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773612.713691:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773612.713700:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773612.713712:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534715776:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773612.713742:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773612.713746:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534715904:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773612.713751:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773612.713756:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773612.713765:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2608 00000800:00000100:0.0:1790773612.713771:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773612.713771:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773612.713788:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773612/real 1790773612] req@ffff9c4efac4f800 x1877759534715904/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773612.713789:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773612.713804:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773612.713810:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534715904/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773612.713821:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773612.713823:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773612.713827:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534715904:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773612.713834:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534716288/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773612.713835:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534716032:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773612.713864:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773612.713865:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773612.713877:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773612.713880:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773612.713881:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773612/real 1790773612] req@ffff9c4efac4c700 x1877759534716032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773612.713897:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2608 00000100:00080000:1.0:1790773612.713908:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773612.713931:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534716032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773612.713931:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773612.713947:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790773612.713953:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773612.713958:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773612.713962:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534716032:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773612.713969:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534716416/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773612.713972:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534716160:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773612.713999:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773612.714014:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773612/real 1790773612] req@ffff9c4efac4ed80 x1877759534716160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773612.714027:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534716160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773612.714033:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773612.714035:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773612.714037:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534716160:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773612.714049:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534716288:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773612.714061:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773612.714068:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773612/real 1790773612] req@ffff9c4efac4dc00 x1877759534716288/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773612.714076:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534716288/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773612.714088:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773612.714090:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773612.714093:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534716288:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773612.714097:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534716416:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773612.714107:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773612.714114:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773612/real 1790773612] req@ffff9c4efac4df80 x1877759534716416/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773612.714137:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534716416/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773612.714148:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773612.714152:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773612.714156:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534716416:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773612.776727:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773612.776752:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773612.776782:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773612.776792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773612.840730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773612.840748:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773612.840909:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773612.968714:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773612.968722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773613.032721:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773613.096716:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773613.160734:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773613.160767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773613.160768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773613.224745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773613.288752:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773613.288753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773613.288774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773613.416733:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773613.416740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773613.416776:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773613.416778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773613.480733:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773613.480772:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773613.480773:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773613.480775:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773613.480791:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773613.480815:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773613.608760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773613.672800:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773613.736735:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773613.800763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773613.800763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773613.800785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773613.800788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773613.864739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773613.864784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773613.864888:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773613.992723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773613.992732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773614.056694:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773614.120742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773614.184772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773614.184772:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773614.184786:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773614.248716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773614.312725:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773614.312725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773614.312733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773614.440762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773614.440762:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773614.440762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773614.440798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773614.504722:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773614.504730:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773614.504752:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773614.504778:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773614.504787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773614.504792:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773614.632804:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773614.696782:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773614.760776:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773614.824839:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773614.824841:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773614.824841:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773614.824857:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773614.888759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773614.888761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773614.888932:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773615.016766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773615.016797:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773615.080764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773615.144780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773615.208750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773615.208753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773615.208808:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773615.272742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773615.336795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773615.336857:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773615.336874:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773615.464773:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773615.464792:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773615.464814:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773615.464821:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773615.528792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773615.528796:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773615.528803:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773615.528843:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773615.528851:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773615.528890:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773615.656779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773615.720781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773615.784743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773615.848785:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773615.848794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773615.848794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773615.848794:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773615.912724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773615.912730:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773615.912886:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773616.040744:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773616.040786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773616.104751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773616.168782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773616.232781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773616.232786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773616.232790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773616.296860:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773616.360722:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773616.360730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773616.360750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773616.488751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773616.488751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773616.488751:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773616.488803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773616.552798:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773616.552799:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773616.552799:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773616.552799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773616.552842:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773616.552853:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773616.680737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773616.744762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773616.808740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773616.872758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773616.872758:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773616.872759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773616.872760:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773616.936740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773616.936752:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773616.936791:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773617.064716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773617.064757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773617.128728:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773617.192744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773617.256758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773617.256758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773617.256759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773617.320766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773617.384741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773617.384746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773617.384754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773617.512730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773617.512730:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773617.512730:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773617.512734:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773617.576735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773617.576740:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773617.576741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773617.576758:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773617.576780:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773617.576802:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773617.704767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773617.768746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773617.832733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773617.832769:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773617.832776:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773617.832781:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773617.832790:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2613 00000100:00080000:2.0:1790773617.832800:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773617.832810:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773617.832845:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534716544/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773617.832869:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773617.832874:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773617.832877:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773617.832881:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2613 00000100:00080000:2.0:1790773617.832885:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773617.832891:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773617.832901:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773617.832910:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534716544:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773617.832915:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534716672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773617.832925:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773617.832928:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773617.832931:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773617.832934:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2613 00000100:00080000:2.0:1790773617.832938:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773617.832943:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773617.832947:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773617.832958:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534716800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773617.832969:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773617.832973:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773617.832977:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773617.832979:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773617.832982:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2613 00000100:00080000:2.0:1790773617.832986:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773617.832991:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773617.832995:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773617.833005:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534716928/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773617.833011:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773617/real 1790773617] req@ffff9c4efbcc2a00 x1877759534716544/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773617.833015:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773617.833019:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773617.833033:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773617.833038:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2613 00000100:00100000:0.0:1790773617.833042:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534716544/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773617.833042:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773617.833047:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773617.833052:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:0.0:1790773617.833054:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1790773617.833063:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534717056/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773617.833067:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773617.833072:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534716544:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773617.833073:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773617.833077:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773617.833079:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773617.833083:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2613 00000100:00080000:2.0:1790773617.833086:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773617.833091:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773617.833095:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773617.833101:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534716672:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773617.833105:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534717184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773617.833115:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773617.833118:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773617.833120:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773617.833125:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773617.833125:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2613 00000100:00080000:2.0:1790773617.833128:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773617.833133:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773617.833137:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773617.833138:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773617/real 1790773617] req@ffff9c4efbcc0a80 x1877759534716672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773617.833147:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534717312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773617.833151:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534716672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773617.833158:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773617.833161:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773617.833163:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773617.833166:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2613 00000100:00080000:0.0:1790773617.833169:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773617.833170:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773617.833173:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773617.833174:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773617.833177:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534716672:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790773617.833179:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773617.833183:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534716800:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773617.833188:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534717440/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773617.833201:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773617.833205:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773617.833208:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773617.833211:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773617.833212:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773617/real 1790773617] req@ffff9c4efbcc0380 x1877759534716800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773617.833225:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534716800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773617.833225:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2613 00000100:00080000:2.0:1790773617.833229:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773617.833233:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773617.833238:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790773617.833242:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773617.833246:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773617.833249:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534717568/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773617.833250:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534716800:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773617.833257:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534716928:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773617.833276:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773617.833289:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773617/real 1790773617] req@ffff9c4efbcc1f80 x1877759534716928/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773617.833302:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534716928/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773617.833309:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773617.833312:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773617.833314:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534716928:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773617.833319:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534717056:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773617.833334:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773617.833341:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773617/real 1790773617] req@ffff9c4efbcc1c00 x1877759534717056/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773617.833349:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534717056/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773617.833355:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773617.833357:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773617.833360:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534717056:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773617.833373:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534717184:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773617.833385:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773617.833391:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773617/real 1790773617] req@ffff9c4efbcc3100 x1877759534717184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773617.833398:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534717184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773617.833409:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773617.833411:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773617.833413:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534717184:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773617.833417:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534717312:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773617.833426:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773617.833432:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773617/real 1790773617] req@ffff9c4efbcc2680 x1877759534717312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773617.833440:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534717312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773617.833446:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773617.833448:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773617.833450:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534717312:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773617.833453:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534717440:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773617.833463:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773617.833469:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773617/real 1790773617] req@ffff9c4efbcc0700 x1877759534717440/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773617.833477:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534717440/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773617.833484:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773617.833486:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773617.833488:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534717440:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773617.833492:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534717568:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773617.833515:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773617.833527:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773617/real 1790773617] req@ffff9c4efbcc1500 x1877759534717568/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773617.833539:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534717568/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773617.833549:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773617.833553:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773617.833557:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534717568:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773617.896768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773617.896768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773617.896768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773617.896772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773617.960701:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773617.960777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773617.960905:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773618.088740:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773618.088741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773618.152755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773618.216751:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773618.280744:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773618.280746:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773618.280769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773618.344749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773618.408765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773618.408765:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773618.408765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773618.536766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773618.536766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773618.536785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773618.536787:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773618.600754:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773618.600754:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773618.600794:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773618.600798:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773618.600816:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773618.600817:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773618.728767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773618.792771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773618.856772:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773618.920749:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773618.920749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773618.920767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773618.920770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773618.984756:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773618.984756:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773618.984888:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773619.112733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773619.112776:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773619.176831:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773619.240725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773619.304855:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773619.304855:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773619.304858:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773619.368703:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773619.432749:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773619.432749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773619.432749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773619.560730:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773619.560730:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773619.560762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773619.560805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773619.624768:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773619.624775:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773619.624809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773619.624809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773619.624821:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773619.624821:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773619.752809:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773619.816740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773619.880724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773619.944767:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773619.944767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773619.944780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773619.944781:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773620.008737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773620.008769:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773620.008902:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773620.136746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773620.136758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773620.200784:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773620.264742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773620.328766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773620.328776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773620.328776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773620.392760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773620.456761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773620.456761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773620.456771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773620.584783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773620.584795:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773620.584799:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773620.584800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773620.648722:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773620.648739:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773620.648787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773620.648793:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773620.648796:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773620.648798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773620.776722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773620.840756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773620.904737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773620.968754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773620.968755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773620.968755:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773620.968764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773621.032711:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773621.032723:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773621.032894:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773621.160722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773621.160744:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773621.224747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773621.288732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773621.352768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773621.352768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773621.352769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773621.416714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773621.480734:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773621.480734:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773621.480734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773621.608759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773621.608760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773621.608760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773621.608796:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773621.672727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773621.672733:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773621.672736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773621.672773:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773621.672778:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773621.672788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773621.800799:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773621.864787:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773621.928714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773621.992781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773621.992781:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773621.992782:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773621.992791:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773622.056763:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773622.056782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773622.056901:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773622.184726:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773622.184728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773622.248772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773622.312714:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773622.376830:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773622.376830:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773622.376830:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773622.440780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773622.504732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773622.504745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773622.504749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773622.632724:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773622.632746:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773622.632801:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773622.632847:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773622.696769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773622.696770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773622.696775:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773622.696780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773622.696804:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773622.696814:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773622.824724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773622.888722:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773622.952746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773622.952798:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773622.952812:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773622.952820:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773622.952837:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2618 00000100:00080000:1.0:1790773622.952871:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773622.952893:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773622.953003:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534717696/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773622.953073:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773622.953094:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773622.953099:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773622.953110:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534717696:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773622.953111:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2618 00000100:00080000:1.0:1790773622.953131:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773622.953145:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773622.953163:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773622.953194:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534717824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773622.953213:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773622.953254:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773622.953263:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773622.953268:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773622.953279:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2618 00000100:00080000:1.0:1790773622.953289:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773622.953293:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773622/real 1790773622] req@ffff9c4efac4e300 x1877759534717696/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773622.953321:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773622.953331:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773622.953332:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534717696/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773622.953354:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773622.953358:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773622.953362:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534717696:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773622.953380:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534717952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773622.953397:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534717824:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773622.953402:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773622.953409:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773622.953413:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773622.953422:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2618 00000800:00000100:0.0:1790773622.953425:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773622.953434:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773622.953444:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773622/real 1790773622] req@ffff9c4efac4df80 x1877759534717824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773622.953451:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773622.953461:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534717824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773622.953465:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00080000:0.0:1790773622.953485:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773622.953491:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773622.953499:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534718080/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773622.953500:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534717824:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773622.953537:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773622.953538:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534717952:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773622.953557:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773622.953562:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773622.953571:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2618 00000100:00080000:1.0:1790773622.953583:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773622.953588:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773622.953601:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773622.953606:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773622/real 1790773622] req@ffff9c4efac4dc00 x1877759534717952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773622.953616:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773622.953641:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534717952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773622.953671:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773622.953675:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773622.953678:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534717952:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773622.953692:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534718080:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773622.953717:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534718208/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773622.953733:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773622.953742:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773622.953750:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773622.953754:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773622.953758:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773622/real 1790773622] req@ffff9c4efac4ed80 x1877759534718080/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773622.953766:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2618 00000100:00080000:1.0:1790773622.953783:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773622.953789:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534718080/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773622.953798:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773622.953814:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773622.953818:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773622.953821:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773622.953831:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534718080:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773622.953853:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534718208:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773622.953855:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534718336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773622.953879:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773622.953880:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773622.953894:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1790773622.953899:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773622/real 1790773622] req@ffff9c4efac4c700 x1877759534718208/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773622.953903:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773622.953942:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534718208/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773622.953966:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2618 00000100:00080000:1.0:1790773622.953984:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773622.953985:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773622.954000:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773622.954008:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534718208:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773622.954008:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773622.954037:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773622.954041:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534718336:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773622.954083:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534718464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773622.954084:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773622.954104:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773622/real 1790773622] req@ffff9c4efac4f800 x1877759534718336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773622.954140:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534718336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773622.954153:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773622.954159:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773622.954162:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773622.954168:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773622.954171:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773622.954175:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534718336:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773622.954204:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534718464:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773622.954209:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2618 00000100:00080000:1.0:1790773622.954220:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773622.954229:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773622.954234:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773622.954244:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773622/real 1790773622] req@ffff9c4efac4ea00 x1877759534718464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773622.954246:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773622.954279:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534718464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773622.954281:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534718592/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773622.954335:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773622.954343:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773622.954345:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773622.954351:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534718464:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773622.954358:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773622.954363:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773622.954369:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534718592:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773622.954375:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2618 00000100:00080000:1.0:1790773622.954385:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773622.954398:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773622.954402:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773622.954410:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773622.954412:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773622/real 1790773622] req@ffff9c4efac4e680 x1877759534718592/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773622.954429:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534718592/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773622.954433:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534718720/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773622.954480:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773622.954487:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773622.954491:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534718592:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773622.954509:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534718720:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773622.954536:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773622.954561:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773622/real 1790773622] req@ffff9c4efac4f480 x1877759534718720/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773622.954570:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534718720/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773622.954578:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773622.954580:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773622.954583:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534718720:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773623.016723:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773623.016734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773623.016739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773623.016815:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773623.080759:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773623.080796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773623.080941:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773623.208718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773623.208718:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773623.272736:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773623.336752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773623.400756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773623.400775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773623.400782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773623.464722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773623.528722:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773623.528728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773623.528759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773623.656735:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773623.656737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773623.656737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773623.656771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773623.720805:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773623.720805:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773623.720812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773623.720819:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773623.720846:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773623.720847:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773623.848757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773623.912741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773623.976764:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773624.040738:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773624.040738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773624.040742:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773624.040780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773624.104778:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773624.104781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773624.104871:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773624.232780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773624.232790:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773624.296750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773624.360710:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773624.424726:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773624.424745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773624.424776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773624.488728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773624.552745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773624.552746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773624.552746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773624.680716:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773624.680718:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773624.680768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773624.680777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773624.744743:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773624.744744:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773624.744753:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773624.744756:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773624.744823:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773624.744824:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773624.872705:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773624.936721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773625.000724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773625.064717:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773625.064728:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773625.064755:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773625.064761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773625.128715:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773625.128750:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773625.128800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773625.256718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773625.256727:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773625.320730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773625.384759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773625.448717:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773625.448760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773625.448761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773625.512775:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773625.576723:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773625.576764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773625.576798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773625.704773:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773625.704773:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773625.704782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773625.704815:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773625.768726:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773625.768743:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773625.768746:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773625.768762:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773625.768778:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773625.768784:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773625.896706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773625.960711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773626.024745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773626.088767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773626.088776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773626.088779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773626.088779:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773626.152776:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773626.152778:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773626.152778:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773626.280721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773626.280749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773626.344750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773626.408757:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773626.472698:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773626.472735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773626.472756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773626.536746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773626.600768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773626.600780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773626.600782:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773626.728750:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773626.728750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773626.728791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773626.728802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773626.792758:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773626.792759:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773626.792759:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773626.792759:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773626.792805:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773626.792823:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773626.920748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773626.984727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773627.048782:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773627.112770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773627.112770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773627.112775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773627.112776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773627.176764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773627.176793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773627.176834:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773627.304774:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773627.304774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773627.368742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773627.432742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773627.496743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773627.496771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773627.496778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773627.560749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773627.624772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773627.624772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773627.624773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773627.752816:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773627.752816:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773627.752817:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773627.752818:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773627.816810:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773627.816811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773627.816812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773627.816813:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773627.816887:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773627.816891:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773627.944774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773628.008790:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773628.072769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773628.072778:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773628.072785:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773628.072790:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773628.072801:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2623 00000100:00080000:3.0:1790773628.072815:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773628.072846:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773628.072897:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534718848/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773628.072924:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773628.072927:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773628.072929:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773628.072932:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2623 00000100:00080000:3.0:1790773628.072937:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773628.072942:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773628.072953:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773628.072954:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534718848:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773628.072970:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534718976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773628.072979:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773628.072982:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773628.072983:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773628.072986:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2623 00000100:00080000:3.0:1790773628.072990:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773628.072993:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773628.072996:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773628.073005:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534719104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773628.073011:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773628.073015:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773628.073017:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773628.073020:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2623 00000100:00080000:3.0:1790773628.073023:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773628.073028:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773628.073029:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773628.073036:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773628.073044:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534719232/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773628.073051:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773628.073055:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773628.073056:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773628.073059:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2623 00000100:00080000:3.0:1790773628.073062:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773628.073065:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773628.073069:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773628.073077:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773628/real 1790773628] req@ffff9c4ef73c0000 x1877759534718848/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773628.073080:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534719360/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773628.073104:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773628.073107:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773628.073109:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773628.073112:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534718848/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773628.073115:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2623 00000100:00080000:3.0:1790773628.073120:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773628.073126:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773628.073131:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773628.073133:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773628.073139:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790773628.073142:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534719488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773628.073145:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534718848:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773628.073159:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773628.073162:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773628.073164:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773628.073168:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2623 00000100:00080000:3.0:1790773628.073170:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773628.073173:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773628.073177:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773628.073179:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534718976:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773628.073187:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534719616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773628.073196:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773628.073199:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773628.073201:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773628.073204:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773628.073208:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2623 00000100:00080000:3.0:1790773628.073213:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773628.073217:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773628.073221:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773628.073222:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773628/real 1790773628] req@ffff9c4ef73c1500 x1877759534718976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773628.073234:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534719744/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773628.073242:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534718976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773628.073245:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773628.073252:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773628.073254:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773628.073263:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2623 00000100:00080000:3.0:1790773628.073268:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773628.073273:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773628.073277:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790773628.073282:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790773628.073285:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534719872/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773628.073292:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773628.073300:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534718976:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773628.073307:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534719104:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773628.073327:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773628.073337:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773628/real 1790773628] req@ffff9c4ef73c0700 x1877759534719104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773628.073346:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534719104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773628.073353:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773628.073355:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773628.073358:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534719104:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773628.073362:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534719232:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773628.073373:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773628.073379:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773628/real 1790773628] req@ffff9c4ef73c2a00 x1877759534719232/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773628.073387:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534719232/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773628.073394:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773628.073396:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773628.073398:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534719232:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773628.073402:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534719360:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773628.073413:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773628.073418:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773628/real 1790773628] req@ffff9c4ef73c2d80 x1877759534719360/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773628.073426:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534719360/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773628.073434:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773628.073436:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773628.073439:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534719360:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773628.073442:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534719488:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773628.073453:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773628.073468:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773628/real 1790773628] req@ffff9c4ef73c3800 x1877759534719488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773628.073476:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534719488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773628.073488:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773628.073490:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773628.073492:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534719488:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773628.073508:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534719616:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773628.073521:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773628.073529:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773628/real 1790773628] req@ffff9c4ef73c0a80 x1877759534719616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773628.073543:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534719616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773628.073552:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773628.073556:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773628.073560:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534719616:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773628.073567:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534719744:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773628.073587:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773628.073600:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773628/real 1790773628] req@ffff9c4ef73c0e00 x1877759534719744/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773628.073614:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534719744/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773628.073622:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773628.073625:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773628.073627:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534719744:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773628.073632:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534719872:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773628.073647:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773628.073674:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773628/real 1790773628] req@ffff9c4ef73c2300 x1877759534719872/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773628.073682:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534719872/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773628.073689:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773628.073692:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773628.073694:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534719872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773628.136788:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773628.136788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773628.136788:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773628.136795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773628.200785:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773628.200789:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773628.200871:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773628.328774:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773628.328777:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773628.392772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773628.456790:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773628.520719:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773628.520736:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773628.520737:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773628.584768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773628.648711:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773628.648741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773628.648760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773628.776754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773628.776754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773628.776768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773628.776768:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773628.840791:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773628.840791:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773628.840796:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773628.840796:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773628.840853:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773628.840872:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773628.968784:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773629.032772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773629.096758:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773629.160745:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773629.160745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773629.160778:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773629.160779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773629.224751:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773629.224780:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773629.224896:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773629.352731:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773629.352734:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773629.416729:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773629.480724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773629.544707:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773629.544707:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773629.544720:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773629.608728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773629.672745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773629.672745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773629.672748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773629.800762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773629.800762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773629.800762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773629.800766:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773629.864731:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773629.864731:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773629.864762:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773629.864887:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773629.864896:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773629.864915:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773629.992761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773630.056829:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773630.120732:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773630.184739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773630.184826:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773630.184833:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773630.184833:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773630.248746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773630.248776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773630.248884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773630.376730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773630.376742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773630.440726:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773630.504802:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773630.568746:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773630.568759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773630.568765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773630.632733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773630.696761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773630.696761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773630.696765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773630.824784:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773630.824804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773630.824805:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773630.824811:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773630.888747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773630.888761:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773630.888786:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773630.888803:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773630.888809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773630.888822:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773631.016752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773631.080778:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773631.144823:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773631.208797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773631.208797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773631.208797:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773631.208806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773631.272784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773631.272784:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773631.272849:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773631.400740:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773631.400740:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773631.464727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773631.528753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773631.592785:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773631.592785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773631.592785:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773631.656717:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773631.720815:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773631.720815:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773631.720827:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773631.848795:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773631.848798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773631.848798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773631.848946:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773631.912729:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773631.912775:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773631.912778:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773631.912779:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773631.912805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773631.912812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773632.040734:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773632.104810:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773632.168787:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773632.232730:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773632.232761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773632.232763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773632.232763:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773632.296774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773632.296775:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773632.296850:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773632.424785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773632.424785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773632.488795:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773632.552766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773632.616731:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773632.616755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773632.616774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773632.680762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773632.744781:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773632.744781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773632.744793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773632.872794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773632.872794:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773632.872806:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773632.872806:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773632.936778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773632.936779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773632.936781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773632.936793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773632.936840:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773632.936848:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773633.064768:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773633.128840:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773633.192796:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773633.192807:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773633.192808:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773633.192818:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773633.192834:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2629 00000100:00080000:3.0:1790773633.192846:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773633.192862:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773633.192909:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534720000/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773633.192944:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773633.192949:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773633.192951:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773633.192956:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2629 00000100:00080000:3.0:1790773633.192960:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773633.192966:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773633.192982:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773633.193005:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534720000:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773633.193005:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534720128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773633.193039:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773633.193043:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773633.193046:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773633.193051:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2629 00000100:00080000:3.0:1790773633.193056:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773633.193063:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773633.193070:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773633.193086:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534720256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773633.193099:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773633.193110:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773633.193116:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773633.193118:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773633.193126:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2629 00000100:00080000:3.0:1790773633.193132:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773633.193140:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773633.193147:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773633.193151:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773633/real 1790773633] req@ffff9c4ef73c1c00 x1877759534720000/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773633.193167:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534720384/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773633.193172:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534720000/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773633.193206:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773633.193212:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773633.193213:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773633.193216:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773633.193224:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773633.193230:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2629 00000100:00100000:0.0:1790773633.193233:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534720000:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773633.193241:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773633.193252:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773633.193259:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773633.193273:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534720512/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773633.193287:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773633.193290:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534720128:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773633.193294:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773633.193300:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773633.193308:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2629 00000100:00080000:3.0:1790773633.193313:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773633.193321:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773633.193328:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773633.193333:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773633.193352:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534720640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773633.193360:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773633/real 1790773633] req@ffff9c4ef73c2300 x1877759534720128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773633.193383:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773633.193394:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773633.193397:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773633.193403:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534720128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773633.193404:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2629 00000100:00080000:3.0:1790773633.193415:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773633.193428:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773633.193437:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773633.193441:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773633.193447:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790773633.193454:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534720768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773633.193456:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534720128:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773633.193474:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534720256:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773633.193498:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773633.193505:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773633.193515:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1790773633.193518:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773633/real 1790773633] req@ffff9c4ef73c0e00 x1877759534720256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773633.193520:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773633.193534:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2629 00000100:00080000:3.0:1790773633.193539:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773633.193550:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773633.193554:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534720256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773633.193566:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773633.193569:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773633.193578:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773633.193586:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534720256:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773633.193594:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534720896/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773633.193603:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534720384:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790773633.193627:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773633.193631:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773633.193635:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773633.193636:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773633.193674:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2629 00000100:00080000:3.0:1790773633.193681:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773633.193689:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773633/real 1790773633] req@ffff9c4ef73c0a80 x1877759534720384/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773633.193692:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773633.193708:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773633.193716:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534720384/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773633.193742:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790773633.193744:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534721024/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773633.193752:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773633.193761:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534720384:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773633.193801:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534720512:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773633.193825:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773633.193833:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773633/real 1790773633] req@ffff9c4ef73c3800 x1877759534720512/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773633.193848:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534720512/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773633.193855:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773633.193858:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773633.193860:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534720512:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773633.193865:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534720640:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773633.193885:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773633.193898:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773633/real 1790773633] req@ffff9c4ef73c2d80 x1877759534720640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773633.193912:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534720640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773633.193930:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773633.193934:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773633.193937:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534720640:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773633.193943:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534720768:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773633.193959:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773633.193971:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773633/real 1790773633] req@ffff9c4ef73c2a00 x1877759534720768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773633.193983:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534720768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773633.194000:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773633.194004:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773633.194008:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534720768:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773633.194015:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534720896:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773633.194032:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773633.194041:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773633/real 1790773633] req@ffff9c4ef73c0700 x1877759534720896/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773633.194049:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534720896/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773633.194057:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773633.194060:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773633.194063:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534720896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773633.194067:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534721024:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773633.194078:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773633.194085:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773633/real 1790773633] req@ffff9c4ef73c1500 x1877759534721024/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773633.194093:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534721024/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773633.194099:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773633.194102:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773633.194104:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534721024:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773633.256801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773633.256801:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773633.256819:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773633.256848:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773633.320781:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773633.320781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773633.320913:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773633.448781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773633.448781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773633.512797:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773633.576784:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773633.640798:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773633.640798:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773633.640801:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773633.704807:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773633.768784:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773633.768788:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773633.768788:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773633.896831:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773633.896831:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773633.896833:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773633.896834:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773633.960798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773633.960798:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773633.960803:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773633.960812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773633.960838:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773633.960860:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773634.088778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773634.152765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773634.216738:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773634.280793:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773634.280793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773634.280793:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773634.280800:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773634.344793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773634.344820:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773634.344884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773634.472778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773634.472778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773634.536792:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773634.600799:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773634.664764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773634.664791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773634.664791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773634.728818:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773634.792794:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773634.792795:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773634.792804:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773634.920767:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773634.920767:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773634.920767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773634.920804:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773634.984808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773634.984814:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773634.984819:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773634.984822:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773634.984856:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773634.984869:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773635.112777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773635.176795:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773635.240781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773635.304808:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773635.304809:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773635.304809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773635.304809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773635.368765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773635.368820:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773635.368868:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773635.496776:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773635.496776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773635.560772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773635.624777:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773635.688805:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773635.688809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773635.688818:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773635.752779:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773635.816758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773635.816759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773635.816793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773635.944814:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773635.944815:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773635.944815:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773635.944815:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773636.008802:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773636.008802:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773636.008802:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773636.008803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773636.008870:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773636.008870:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773636.136737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773636.200776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773636.264741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773636.328778:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773636.328778:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773636.328779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773636.328779:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773636.392751:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773636.392754:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773636.392800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773636.520744:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773636.520744:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773636.584754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773636.648719:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773636.712726:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773636.712726:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773636.712755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773636.776749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773636.840730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773636.840730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773636.840739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773636.968735:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773636.968735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773636.968745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773636.968745:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773637.032718:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773637.032725:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773637.032738:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773637.032739:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773637.032762:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773637.032764:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773637.160711:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773637.224738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773637.288718:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773637.352770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773637.352770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773637.352770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773637.352771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773637.416782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773637.416901:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773637.416936:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773637.544736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773637.544736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773637.608746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773637.672721:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773637.736725:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773637.736725:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773637.736735:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773637.800704:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773637.864720:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773637.864720:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773637.864724:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773637.992731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773637.992731:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773637.992732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773637.992844:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773638.056763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773638.056764:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773638.056770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773638.056888:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773638.056908:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773638.056919:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773638.184741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773638.248750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773638.312760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773638.312779:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773638.312786:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773638.312791:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773638.312802:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2634 00000100:00080000:2.0:1790773638.312811:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773638.312831:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773638.312870:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534721152/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773638.312897:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773638.312902:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773638.312904:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773638.312908:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2634 00000100:00080000:2.0:1790773638.312913:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773638.312919:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773638.312929:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773638.312930:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534721152:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773638.312954:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534721280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773638.312969:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773638.312973:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773638.312975:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773638.312980:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2634 00000100:00080000:2.0:1790773638.312985:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773638.312991:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773638.312996:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773638.313003:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773638.313011:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534721408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773638.313022:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773638.313027:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773638.313029:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773638.313034:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2634 00000100:00080000:2.0:1790773638.313039:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773638.313045:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773638.313049:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773638/real 1790773638] req@ffff9c4efbcc1880 x1877759534721152/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773638.313053:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773638.313071:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534721536/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773638.313073:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534721152/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773638.313101:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773638.313103:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773638.313110:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773638.313113:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773638.313115:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773638.313121:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534721152:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773638.313126:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2634 00000100:00080000:2.0:1790773638.313132:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773638.313138:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773638.313144:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773638.313151:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534721280:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773638.313157:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534721664/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773638.313170:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773638.313176:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773638.313176:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773638.313181:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773638.313188:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2634 00000100:00100000:0.0:1790773638.313193:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773638/real 1790773638] req@ffff9c4efbcc2300 x1877759534721280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773638.313194:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773638.313204:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773638.313210:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534721280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773638.313211:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773638.313227:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534721792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773638.313235:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773638.313239:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773638.313243:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773638.313245:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773638.313251:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2634 00000100:00080000:0.0:1790773638.313253:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773638.313256:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773638.313262:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534721280:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773638.313266:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773638.313274:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773638.313276:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534721408:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773638.313287:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534721920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773638.313307:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773638.313307:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773638.313314:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773638.313317:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773638.313324:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2634 00000100:00100000:0.0:1790773638.313327:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773638/real 1790773638] req@ffff9c4efbcc1500 x1877759534721408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773638.313329:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773638.313339:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773638.313343:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534721408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773638.313347:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773638.313355:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773638.313358:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773638.313364:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534721408:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773638.313366:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534722048/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773638.313372:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534721536:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773638.313387:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773638.313390:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773638.313393:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773638.313398:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773638.313402:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2634 00000100:00080000:2.0:1790773638.313408:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773638.313409:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773638/real 1790773638] req@ffff9c4efbcc0700 x1877759534721536/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773638.313417:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773638.313422:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534721536/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773638.313430:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773638.313434:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773638.313434:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773638.313440:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534721536:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773638.313453:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534722176/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773638.313459:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534721664:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773638.313481:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773638.313503:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773638/real 1790773638] req@ffff9c4efbcc2680 x1877759534721664/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773638.313516:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534721664/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773638.313527:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773638.313532:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773638.313536:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534721664:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773638.313543:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534721792:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773638.313560:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773638.313569:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773638/real 1790773638] req@ffff9c4efbcc3100 x1877759534721792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773638.313577:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534721792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773638.313589:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773638.313591:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773638.313593:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534721792:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773638.313598:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534721920:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773638.313608:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773638.313614:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773638/real 1790773638] req@ffff9c4efbcc1c00 x1877759534721920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773638.313622:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534721920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773638.313701:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773638.313704:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773638.313707:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534721920:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773638.313711:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534722048:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773638.313728:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773638.313737:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773638/real 1790773638] req@ffff9c4efbcc1f80 x1877759534722048/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773638.313745:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534722048/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773638.313752:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773638.313755:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773638.313758:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534722048:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773638.313775:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534722176:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773638.313788:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773638.313795:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773638/real 1790773638] req@ffff9c4efbcc0380 x1877759534722176/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773638.313803:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534722176/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773638.313809:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773638.313811:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773638.313814:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534722176:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773638.376727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773638.376728:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773638.376729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773638.376735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773638.440743:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773638.440752:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773638.440775:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773638.568711:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773638.568730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773638.632755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773638.696761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773638.760747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773638.760747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773638.760748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773638.824709:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773638.888728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773638.888732:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773638.888734:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773639.016719:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773639.016719:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773639.016721:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773639.016729:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773639.080728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773639.080730:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773639.080738:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773639.080739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773639.080768:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773639.080770:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773639.208725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773639.272726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773639.336714:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773639.400723:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773639.400723:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773639.400724:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773639.400724:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773639.464732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773639.464742:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773639.464815:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773639.592734:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773639.592834:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773639.656733:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773639.720755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773639.784747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773639.784764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773639.784778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773639.848726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773639.912821:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773639.912822:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773639.912825:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773640.040750:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773640.040752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773640.040754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773640.040757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773640.104778:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773640.104778:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773640.104781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773640.104812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773640.104915:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773640.104923:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773640.232755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773640.296756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773640.360752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773640.424737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773640.424738:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773640.424738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773640.424787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773640.488706:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773640.488750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773640.488951:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773640.616719:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773640.616753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773640.680761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773640.744764:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773640.808782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773640.808787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773640.808787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773640.872718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773640.936765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773640.936771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773640.936777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773641.064739:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773641.064740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773641.064746:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773641.064783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773641.128770:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773641.128778:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773641.128783:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773641.128783:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773641.128828:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773641.128834:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773641.256746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773641.320756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773641.384730:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773641.448775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773641.448775:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773641.448776:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773641.448782:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773641.512766:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773641.512799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773641.512814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773641.640765:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773641.640765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773641.704755:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773641.768741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773641.832763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773641.832804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773641.832816:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773641.896768:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773641.960766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773641.960766:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773641.960791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773642.088733:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773642.088746:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773642.088749:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773642.088791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773642.152726:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773642.152750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773642.152772:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773642.152785:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773642.152798:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773642.152801:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773642.280760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773642.344752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773642.408742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773642.472757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773642.472757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773642.472768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773642.472768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773642.536806:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773642.536816:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773642.536866:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773642.664787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773642.664787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773642.728764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773642.792800:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773642.856801:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773642.856802:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773642.856813:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773642.920744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773642.984725:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773642.984772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773642.984772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773643.112760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773643.112770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773643.112770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773643.113695:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773643.176752:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773643.176753:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773643.176766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773643.176766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773643.176808:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773643.176834:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773643.304748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773643.368730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773643.432734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773643.432792:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773643.432801:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773643.432805:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773643.432815:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2639 00000100:00080000:3.0:1790773643.432824:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773643.432835:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773643.432868:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534722304/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773643.432890:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773643.432893:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773643.432895:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773643.432898:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2639 00000100:00080000:3.0:1790773643.432901:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773643.432904:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773643.432912:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773643.432923:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534722432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773643.432930:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534722304:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773643.432942:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773643.432946:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773643.432947:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773643.432951:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2639 00000100:00080000:3.0:1790773643.432954:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773643.432957:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773643.432961:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773643.432970:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534722560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773643.432977:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773643.432981:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773643.432983:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773643.432986:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2639 00000100:00080000:3.0:1790773643.432989:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773643.432993:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773643.432996:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773643.433007:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534722688/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773643.433019:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773643.433020:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773643.433027:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773643.433030:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773643.433038:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2639 00000100:00080000:3.0:1790773643.433044:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773643.433053:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773643.433059:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773643.433074:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534722816/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773643.433085:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773643/real 1790773643] req@ffff9c4ef73c0000 x1877759534722304/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773643.433088:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773643.433101:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773643.433108:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773643.433119:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2639 00000100:00080000:3.0:1790773643.433124:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773643.433129:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534722304/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773643.433137:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773643.433148:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773643.433149:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773643.433157:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773643.433163:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534722304:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773643.433169:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534722944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773643.433183:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773643.433187:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773643.433190:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773643.433195:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534722432:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773643.433200:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2639 00000100:00080000:3.0:1790773643.433208:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773643.433217:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773643.433222:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773643.433230:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773643.433246:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534723072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773643.433248:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773643/real 1790773643] req@ffff9c4ef73c1500 x1877759534722432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773643.433292:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773643.433297:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773643.433298:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773643.433301:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534722432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773643.433310:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2639 00000100:00080000:3.0:1790773643.433318:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773643.433329:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773643.433330:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773643.433338:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773643.433342:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534722432:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790773643.433344:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773643.433351:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534722560:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773643.433363:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534723200/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773643.433374:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773643.433375:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773643.433383:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773643.433385:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773643.433390:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2639 00000100:00080000:3.0:1790773643.433396:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773643.433400:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773643/real 1790773643] req@ffff9c4ef73c0700 x1877759534722560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773643.433406:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773643.433422:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773643.433432:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534722560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773643.433440:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534723328/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773643.433467:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773643.433476:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773643.433481:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534722560:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773643.433487:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534722688:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773643.433504:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773643.433512:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773643/real 1790773643] req@ffff9c4ef73c2a00 x1877759534722688/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773643.433520:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534722688/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773643.433543:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773643.433546:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773643.433549:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534722688:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773643.433554:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534722816:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773643.433571:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773643.433579:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773643/real 1790773643] req@ffff9c4ef73c2d80 x1877759534722816/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773643.433587:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534722816/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773643.433594:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773643.433597:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773643.433600:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534722816:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773643.433604:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534722944:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773643.433615:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773643.433622:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773643/real 1790773643] req@ffff9c4ef73c3800 x1877759534722944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773643.433629:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534722944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773643.433641:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773643.433644:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773643.433646:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534722944:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773643.433682:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534723072:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773643.433696:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773643.433704:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773643/real 1790773643] req@ffff9c4ef73c0a80 x1877759534723072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773643.433718:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534723072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773643.433724:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773643.433726:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773643.433728:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534723072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773643.433732:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534723200:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773643.433743:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773643.433749:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773643/real 1790773643] req@ffff9c4ef73c0e00 x1877759534723200/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773643.433757:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534723200/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773643.433764:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773643.433766:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773643.433768:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534723200:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773643.433773:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534723328:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773643.433783:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773643.433789:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773643/real 1790773643] req@ffff9c4ef73c2300 x1877759534723328/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773643.433797:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534723328/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773643.433811:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773643.433814:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773643.433819:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534723328:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773643.496727:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773643.496739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773643.496765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773643.496768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773643.560752:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773643.560775:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773643.560848:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773643.688718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773643.688762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773643.752784:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773643.816833:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773643.880780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773643.880780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773643.880793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773643.944726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773644.008766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773644.008769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773644.008797:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773644.136729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773644.136747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773644.136752:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773644.136781:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773644.200764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773644.200764:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773644.200764:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773644.200808:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773644.200836:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773644.200842:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773644.328732:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773644.392761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773644.456763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773644.520746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773644.520774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773644.520793:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773644.520800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773644.584732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773644.584785:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773644.584824:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773644.712759:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773644.712801:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773644.776769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773644.840732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773644.904829:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773644.904829:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773644.904831:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773644.968771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773645.032756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773645.032756:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773645.032757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773645.160783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773645.160784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773645.160784:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773645.160784:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773645.224760:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773645.224760:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773645.224760:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773645.224762:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773645.224795:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773645.224796:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773645.352756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773645.416801:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773645.480780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773645.544778:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773645.544781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773645.544782:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773645.544790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773645.608769:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773645.608769:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773645.608839:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773645.736779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773645.736779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773645.800801:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773645.864770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773645.928756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773645.928756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773645.928757:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773645.992762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773646.056698:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773646.056713:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773646.056726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773646.184751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773646.184752:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773646.184752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773646.184752:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773646.248751:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773646.248763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773646.248764:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773646.248764:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773646.248793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773646.248796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773646.376713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773646.440765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773646.504771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773646.568734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773646.568747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773646.568784:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773646.568785:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773646.632718:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773646.632785:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773646.632836:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773646.760719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773646.760762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773646.824759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773646.888759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773646.952779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773646.952794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773646.952795:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773647.016773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773647.080768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773647.080769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773647.080845:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773647.208837:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773647.208840:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773647.208840:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773647.208843:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773647.272744:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773647.272745:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773647.272795:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773647.272795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773647.272815:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773647.272824:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773647.400840:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773647.464767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773647.528796:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773647.592824:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773647.592824:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773647.592825:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773647.592825:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773647.656786:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773647.656794:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773647.656830:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773647.784724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773647.784757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773647.848767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773647.912740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773647.976777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773647.976779:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773647.976788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773648.040766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773648.104806:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773648.104824:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773648.104825:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773648.232772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773648.232772:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773648.232772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773648.232796:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773648.296791:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773648.296791:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773648.296791:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773648.296791:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773648.296875:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773648.296876:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773648.424747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773648.488749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773648.552773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773648.552809:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773648.552813:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773648.552817:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773648.552825:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2644 00000100:00080000:0.0:1790773648.552833:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773648.552842:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773648.552878:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534723456/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773648.552912:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773648.552916:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773648.552917:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773648.552921:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2644 00000100:00080000:0.0:1790773648.552925:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773648.552930:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773648.552939:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773648.552953:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534723584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773648.552961:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773648.552964:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773648.552965:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773648.552969:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2644 00000100:00080000:0.0:1790773648.552971:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773648.552974:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773648.552978:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773648.552989:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534723712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773648.552991:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534723456:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773648.553005:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773648.553011:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773648.553013:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773648.553018:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2644 00000100:00080000:0.0:1790773648.553023:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773648.553031:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773648.553038:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773648.553052:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534723840/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773648.553067:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773648.553071:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773648.553073:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773648.553078:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2644 00000100:00080000:0.0:1790773648.553085:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773648.553091:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773648.553097:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:1.0:1790773648.553099:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773648.553118:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534723968/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773648.553132:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773648.553136:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773648.553139:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773648.553144:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2644 00000100:00080000:0.0:1790773648.553150:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773648.553157:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773648.553164:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773648.553176:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773648/real 1790773648] req@ffff9c4ec7a7f480 x1877759534723456/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773648.553181:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534724096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773648.553229:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773648.553232:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773648.553234:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773648.553241:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2644 00000100:00100000:1.0:1790773648.553242:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534723456/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773648.553249:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773648.553265:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773648.553274:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773648.553281:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773648.553289:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534724224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773648.553295:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773648.553305:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534723456:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773648.553316:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773648.553319:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534723584:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773648.553321:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773648.553331:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773648.553342:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2644 00000100:00080000:0.0:1790773648.553354:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773648.553360:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773648.553373:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773648.553382:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773648.553382:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773648/real 1790773648] req@ffff9c4ec7a7df80 x1877759534723584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773648.553409:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534724352/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773648.553441:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534723584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773648.553444:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773648.553460:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773648.553466:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773648.553477:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2644 00000100:00080000:0.0:1790773648.553483:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773648.553491:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773648.553498:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773648.553501:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773648.553514:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534724480/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773648.553514:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773648.553531:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534723584:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773648.553583:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534723712:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773648.553622:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773648.553637:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773648/real 1790773648] req@ffff9c4ec7a7ed80 x1877759534723712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773648.553684:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534723712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773648.553696:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773648.553701:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773648.553705:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534723712:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773648.553713:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534723840:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773648.553738:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773648.553751:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773648/real 1790773648] req@ffff9c4ec7a7d880 x1877759534723840/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773648.553766:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534723840/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773648.553777:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773648.553782:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773648.553786:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534723840:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773648.553794:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534723968:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773648.553813:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773648.553824:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773648/real 1790773648] req@ffff9c4ec7a7c380 x1877759534723968/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773648.553847:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534723968/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773648.553858:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773648.553862:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773648.553866:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534723968:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773648.553874:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534724096:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773648.553896:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773648.553908:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773648/real 1790773648] req@ffff9c4ec7a7ca80 x1877759534724096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773648.553922:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534724096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773648.553940:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773648.553945:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773648.553949:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534724096:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773648.553957:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534724224:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773648.553976:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773648.553987:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773648/real 1790773648] req@ffff9c4ec7a7fb80 x1877759534724224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773648.554000:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534724224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773648.554009:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773648.554013:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773648.554018:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534724224:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773648.554025:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534724352:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773648.554049:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773648.554062:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773648/real 1790773648] req@ffff9c4ec7a7e300 x1877759534724352/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773648.554075:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534724352/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773648.554087:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773648.554091:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773648.554095:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534724352:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773648.554103:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534724480:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773648.554123:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773648.554133:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773648/real 1790773648] req@ffff9c4ec7a7c700 x1877759534724480/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773648.554146:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534724480/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773648.554157:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773648.554162:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773648.554166:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534724480:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773648.616786:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773648.616787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773648.616787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773648.616793:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773648.680790:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773648.680790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773648.680910:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773648.808779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773648.808779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773648.872753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773648.936737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773649.000748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773649.000748:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773649.000759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773649.064792:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773649.128779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773649.128779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773649.128779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773649.256778:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773649.256779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773649.256779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773649.256783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773649.320767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773649.320771:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773649.320806:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773649.320808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773649.320844:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773649.320846:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773649.448769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773649.512742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773649.576740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773649.640730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773649.640730:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773649.640734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773649.640830:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773649.704777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773649.704788:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773649.704797:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773649.832768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773649.832768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773649.896727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773649.960733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773650.024735:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773650.024735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773650.024735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773650.088722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773650.152765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773650.152765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773650.152852:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773650.280745:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773650.280745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773650.280745:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773650.280745:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773650.344759:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773650.344760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773650.344760:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773650.344898:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773650.344926:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773650.344961:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773650.472750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773650.536730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773650.600725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773650.664717:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773650.664752:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773650.664752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773650.664752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773650.728737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773650.728826:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773650.728868:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773650.856804:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773650.856805:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773650.920754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773650.984754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773651.048779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773651.048779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773651.048786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773651.112798:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773651.176745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773651.176745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773651.176770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773651.304773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773651.304774:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773651.304774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773651.304776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773651.368740:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773651.368751:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773651.368757:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773651.368770:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773651.368810:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773651.368819:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773651.496736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773651.560779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773651.624746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773651.688734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773651.688738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773651.688764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773651.688765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773651.752765:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773651.752858:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773651.752864:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773651.880728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773651.880728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773651.944753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773652.008735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773652.072772:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773652.072772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773652.072786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773652.136747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773652.200758:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773652.200758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773652.200758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773652.328787:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773652.328791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773652.328792:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773652.328798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773652.392777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773652.392777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773652.392806:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773652.392812:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773652.392837:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773652.392840:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773652.520722:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773652.584781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773652.648775:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773652.712794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773652.712795:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773652.712796:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773652.712846:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773652.776826:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773652.776884:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773652.776901:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773652.904730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773652.904730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773652.968719:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773653.032728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773653.096763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773653.096763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773653.096778:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773653.160780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773653.224745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773653.224745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773653.224745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773653.352814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773653.352833:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773653.352837:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773653.352844:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773653.416732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773653.416744:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773653.416747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773653.416784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773653.416803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773653.416809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773653.544746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773653.608782:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773653.672764:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773653.672810:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773653.672818:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773653.672823:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773653.672832:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2649 00000100:00080000:1.0:1790773653.672841:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773653.672852:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773653.672887:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534724608/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773653.672915:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773653.672918:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773653.672920:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773653.672923:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2649 00000100:00080000:1.0:1790773653.672927:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773653.672930:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773653.672937:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773653.672948:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534724736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773653.672957:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773653.672961:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534724608:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773653.672963:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773653.672969:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773653.672978:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2649 00000100:00080000:1.0:1790773653.672980:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773653.672983:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773653.672988:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773653.672999:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534724864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773653.673011:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773653.673014:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773653.673016:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773653.673020:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2649 00000100:00080000:1.0:1790773653.673036:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773653.673040:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773653.673045:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773653.673054:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534724992/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773653.673060:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773653.673063:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773653.673065:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773653.673069:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2649 00000100:00080000:1.0:1790773653.673072:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773653.673077:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773653.673081:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773653.673092:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534725120/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773653.673099:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534724736:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773653.673105:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773653.673109:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773653.673112:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773653.673117:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2649 00000100:00080000:1.0:1790773653.673121:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773653.673131:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773653.673138:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773653.673144:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534724864:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773653.673151:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534725248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773653.673159:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773653.673162:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773653.673165:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773653.673169:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2649 00000100:00080000:1.0:1790773653.673173:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773653.673178:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773653.673183:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773653.673190:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534724992:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773653.673197:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534725376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773653.673205:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773653.673208:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773653.673209:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773653.673213:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2649 00000100:00080000:1.0:1790773653.673217:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773653.673221:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773653.673225:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773653.673233:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534725504/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773653.673246:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773653.673248:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534725120:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773653.673250:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773653.673255:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773653.673263:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2649 00000100:00080000:1.0:1790773653.673266:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773653.673272:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773653.673277:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773653.673291:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534725632/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773653.673300:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534725248:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773653.673350:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534725376:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773653.673382:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534725504:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773653.673407:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534725632:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773653.736730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773653.736740:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773653.736798:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773653.736804:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773653.800764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773653.800766:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773653.800823:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773653.928740:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773653.928741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773653.992744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773654.056742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773654.120724:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773654.120725:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773654.120725:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773654.184745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773654.248754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773654.248754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773654.248754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773654.376714:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773654.376733:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773654.376738:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773654.376738:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773654.440719:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773654.440725:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773654.440730:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773654.440740:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773654.440756:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773654.440765:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773654.568712:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773654.632762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773654.696777:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773654.760751:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773654.760751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773654.760753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773654.760777:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773654.824732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773654.824737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773654.824784:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773654.952725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773654.952762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773655.016759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773655.080759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773655.144730:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773655.144730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773655.144730:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773655.208730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773655.272733:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773655.272733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773655.272768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773655.400776:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773655.400779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773655.400781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773655.400791:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773655.464753:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773655.464753:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773655.464754:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773655.464782:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773655.464812:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773655.464816:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773655.592762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773655.656714:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773655.720768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773655.784770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773655.784770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773655.784770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773655.784775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773655.848780:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773655.848780:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773655.848906:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773655.976776:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773655.976776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773656.040735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773656.104737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773656.168769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773656.168769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773656.168780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773656.232771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773656.296754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773656.296754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773656.296754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773656.424734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773656.424734:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773656.424737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773656.424740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773656.488733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773656.488737:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773656.488743:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773656.488744:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773656.488773:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773656.488776:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773656.616733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773656.680756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773656.744788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:3.0:1790773656.744890:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:3.0:1790773656.744933:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1790773656.744951:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773656.744990:0:2385:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:3.0:1790773656.745006:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773656.745016:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773656.745023:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773656.745028:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773653/real 1790773656] req@ffff9c4efac4df80 x1877759534725632/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1790773656.745033:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773656.745047:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773656.745060:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534725632/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1790773656.745066:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:0.0:1790773656.745078:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000800:00000100:3.0:1790773656.745085:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:0.0:1790773656.745087:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773656.745093:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534725632:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773656.745101:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773653/real 1790773656] req@ffff9c4efac4f800 x1877759534725120/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773656.745124:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534725120/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773656.745137:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773656.745140:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773656.745144:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534725120:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773656.745152:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773653/real 1790773656] req@ffff9c4efac4c700 x1877759534725248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773656.745164:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534725248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773656.745189:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773656.745194:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773656.745198:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534725248:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773656.745205:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773653/real 1790773656] req@ffff9c4efac4ed80 x1877759534725376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773656.745218:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534725376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773656.745227:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773656.745231:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773656.745235:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534725376:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773656.745241:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773653/real 1790773656] req@ffff9c4efac4f480 x1877759534724736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773656.745255:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534724736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773656.745285:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773656.745290:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773656.745295:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534724736:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773656.745302:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773653/real 1790773656] req@ffff9c4efac4e680 x1877759534724864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773656.745321:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534724864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773656.745331:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773656.745335:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773656.745338:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534724864:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773656.745345:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773653/real 1790773656] req@ffff9c4efac4ea00 x1877759534724992/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773656.745358:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534724992/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773656.745369:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773656.745374:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773656.745378:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534724992:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773656.745390:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773653/real 1790773656] req@ffff9c4efac4fb80 x1877759534724608/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773656.745409:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534724608/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773656.745417:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773656.745421:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773656.745425:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534724608:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773656.745461:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773653/real 1790773656] req@ffff9c4efac4dc00 x1877759534725504/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773656.745475:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534725504/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773656.745486:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773656.745491:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773656.745494:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534725504:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773656.808771:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773656.808774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773656.808782:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773656.808782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773656.872715:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773656.872764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773656.872774:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773657.000798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773657.000798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773657.064828:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773657.128746:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773657.192767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773657.192773:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773657.192793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773657.256777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773657.320733:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773657.320733:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773657.320733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773657.448758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773657.448758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773657.448758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773657.448768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773657.512735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773657.512735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773657.512735:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773657.512774:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773657.512781:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773657.512804:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773657.640755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773657.704761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773657.768792:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773657.832783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773657.832783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773657.832794:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773657.832794:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773657.896753:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773657.896829:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773657.896866:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773658.024738:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773658.024738:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773658.088722:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773658.152734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773658.216756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773658.216757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773658.216757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773658.280719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773658.344738:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773658.344741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773658.344775:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773658.472723:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773658.472723:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773658.472727:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773658.472763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773658.536737:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773658.536737:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773658.536748:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773658.536780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773658.536780:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773658.536801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773658.664768:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773658.728774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773658.792770:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773658.792784:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773658.792789:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773658.792812:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2654 00000100:00080000:3.0:1790773658.792830:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773658.792848:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773658.792860:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773658.792902:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534725760/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773658.792935:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773658.792939:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773658.792941:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773658.792944:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2654 00000100:00080000:3.0:1790773658.792948:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773658.792953:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773658.792964:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773658.792981:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534725888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773658.792986:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534725760:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773658.792998:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773658.793003:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773658.793005:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773658.793009:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2654 00000100:00080000:3.0:1790773658.793011:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773658.793015:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773658.793020:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773658.793035:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534726016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773658.793047:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773658.793052:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773658.793054:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773658.793060:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2654 00000100:00080000:3.0:1790773658.793066:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773658.793074:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773658.793086:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790773658.793088:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773658.793109:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534726144/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773658.793122:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773658.793126:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773658.793129:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773658.793136:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2654 00000100:00080000:3.0:1790773658.793140:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773658.793150:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773658/real 1790773658] req@ffff9c4ef73c1c00 x1877759534725760/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773658.793150:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773658.793161:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773658.793185:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534726272/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773658.793202:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534725760/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773658.793202:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773658.793220:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773658.793242:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773658.793248:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773658.793254:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2654 00000100:00080000:0.0:1790773658.793260:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773658.793270:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773658.793278:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534725760:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773658.793287:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773658.793295:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773658.793316:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534726400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773658.793322:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534725888:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773658.793357:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773658.793362:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773658.793364:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773658.793374:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2654 00000800:00000100:0.0:1790773658.793378:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773658.793383:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773658.793395:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773658.793402:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773658.793418:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534726528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773658.793423:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773658/real 1790773658] req@ffff9c4ef73c2300 x1877759534725888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773658.793449:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773658.793456:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773658.793461:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773658.793469:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534725888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773658.793471:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2654 00000100:00080000:3.0:1790773658.793478:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773658.793489:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773658.793502:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:0.0:1790773658.793509:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773658.793519:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790773658.793524:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534726656/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773658.793530:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534725888:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773658.793541:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773658.793546:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773658.793551:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773658.793561:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2654 00000100:00100000:0.0:1790773658.793564:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534726016:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773658.793569:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773658.793582:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773658.793590:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:0.0:1790773658.793608:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773658.793614:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534726784/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773658.793630:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773658/real 1790773658] req@ffff9c4ef73c0e00 x1877759534726016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773658.793648:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534726016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773658.793698:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773658.793704:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773658.793710:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534726016:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773658.793720:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534726144:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773658.793746:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773658.793759:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773658/real 1790773658] req@ffff9c4ef73c0a80 x1877759534726144/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773658.793773:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534726144/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773658.793784:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773658.793789:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773658.793793:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534726144:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773658.793800:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534726272:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773658.793824:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773658.793836:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773658/real 1790773658] req@ffff9c4ef73c3800 x1877759534726272/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773658.793858:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534726272/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773658.793868:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773658.793873:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773658.793877:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534726272:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773658.793900:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534726400:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773658.793923:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773658.793936:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773658/real 1790773658] req@ffff9c4ef73c2d80 x1877759534726400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773658.793949:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534726400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773658.793967:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773658.793972:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773658.793976:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534726400:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773658.793983:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534726528:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773658.794013:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773658.794026:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773658/real 1790773658] req@ffff9c4ef73c2a00 x1877759534726528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773658.794040:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534726528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773658.794056:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773658.794060:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773658.794064:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534726528:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773658.794072:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534726656:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773658.794094:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773658.794106:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773658/real 1790773658] req@ffff9c4ef73c0700 x1877759534726656/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773658.794120:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534726656/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773658.794132:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773658.794136:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773658.794140:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534726656:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773658.794147:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534726784:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773658.794169:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773658.794182:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773658/real 1790773658] req@ffff9c4ef73c1500 x1877759534726784/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773658.794195:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534726784/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773658.794206:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773658.794210:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773658.794214:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534726784:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773658.856824:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773658.856825:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773658.856826:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773658.856827:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773658.920793:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773658.920820:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773658.920937:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773659.048762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773659.048778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773659.112766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773659.176785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773659.240727:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773659.240727:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773659.240751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773659.304752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773659.368732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773659.368745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773659.368746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773659.496742:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773659.496743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773659.496792:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773659.496793:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773659.560722:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773659.560722:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773659.560774:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773659.560778:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773659.560793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773659.560813:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773659.688764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773659.752745:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773659.816861:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773659.880812:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773659.880819:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773659.880819:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773659.880820:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773659.944732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773659.944795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773659.944879:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773660.072741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773660.072778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773660.136777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773660.200807:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773660.264768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773660.264768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773660.264776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773660.328729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773660.392736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773660.392736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773660.392736:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773660.520748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773660.520748:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773660.520748:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773660.520754:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773660.584735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773660.584736:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773660.584736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773660.584753:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773660.584769:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773660.584781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773660.712736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773660.776715:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773660.840713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773660.904759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773660.904760:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773660.904760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773660.904766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773660.968736:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773660.968736:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773660.968792:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773661.096702:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773661.096721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773661.160758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773661.224716:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773661.288754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773661.288757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773661.288782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773661.352767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773661.416755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773661.416755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773661.416755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773661.544742:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773661.544742:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773661.544743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773661.544743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773661.608744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773661.608747:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773661.608763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773661.608846:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773661.608866:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773661.608889:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773661.736773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773661.800748:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773661.864755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773661.928761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773661.928762:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773661.928762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773661.928762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773661.992733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773661.992774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773661.992804:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773662.120701:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773662.120715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773662.184765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773662.248724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773662.312758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773662.312758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773662.312758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773662.376705:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773662.440726:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773662.440727:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773662.440732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773662.568757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773662.568758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773662.568758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773662.568758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773662.632724:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773662.632750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773662.632755:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773662.632756:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773662.632785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773662.632789:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773662.760764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773662.824750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773662.888715:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773662.952755:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773662.952755:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773662.952755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773662.952755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773663.016759:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773663.016784:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773663.016814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773663.144740:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773663.144740:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773663.208771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773663.272724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773663.336759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773663.336765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773663.336767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773663.400766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773663.464762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773663.464762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773663.464762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773663.592731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773663.592732:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773663.592733:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773663.592734:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773663.656734:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773663.656748:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773663.656752:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773663.656765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773663.656777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773663.656780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773663.784793:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773663.848759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773663.912739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773663.912740:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773663.912751:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773663.912756:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773663.912765:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2659 00000100:00080000:3.0:1790773663.912774:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773663.912784:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773663.912818:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534726912/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773663.912844:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773663.912847:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773663.912849:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773663.912852:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2659 00000100:00080000:3.0:1790773663.912855:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773663.912858:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773663.912866:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773663.912877:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534727040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773663.912886:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534726912:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773663.912888:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773663.912895:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773663.912899:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773663.912911:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2659 00000100:00080000:3.0:1790773663.912915:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773663.912918:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773663.912922:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773663.912933:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534727168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773663.912941:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773663.912944:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773663.912945:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773663.912948:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2659 00000100:00080000:3.0:1790773663.912951:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773663.912954:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773663.912959:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:0.0:1790773663.912962:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773663.912970:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534727296/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773663.912980:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773663.912983:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773663.912985:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773663.912987:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2659 00000100:00080000:3.0:1790773663.912990:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773663.912994:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773663.912998:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773663.913011:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534727424/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773663.913013:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773663/real 1790773663] req@ffff9c4ef73c0000 x1877759534726912/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773663.913026:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773663.913032:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773663.913035:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773663.913039:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2659 00000100:00080000:3.0:1790773663.913043:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773663.913050:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773663.913051:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534726912/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773663.913054:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773663.913064:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534727552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773663.913067:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773663.913077:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773663.913080:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773663.913082:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773663.913085:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773663.913089:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2659 00000100:00080000:3.0:1790773663.913093:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773663.913094:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534726912:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773663.913101:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773663.913107:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773663.913117:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534727040:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773663.913144:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534727680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773663.913159:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773663.913164:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773663.913165:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773663.913169:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773663.913175:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2659 00000100:00080000:3.0:1790773663.913180:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773663.913185:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773663.913190:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773663/real 1790773663] req@ffff9c4ef73c1500 x1877759534727040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773663.913192:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773663.913214:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534727040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773663.913214:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534727808/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773663.913249:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773663.913252:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773663.913253:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773663.913256:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773663.913259:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773663.913264:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534727040:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773663.913269:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534727168:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773663.913272:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2659 00000100:00080000:3.0:1790773663.913277:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773663.913285:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773663.913291:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:0.0:1790773663.913306:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773663.913307:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534727936/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773663.913327:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773663/real 1790773663] req@ffff9c4ef73c0700 x1877759534727168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773663.913343:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534727168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773663.913350:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773663.913352:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773663.913355:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534727168:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773663.913360:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534727296:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773663.913381:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773663.913392:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773663/real 1790773663] req@ffff9c4ef73c2a00 x1877759534727296/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773663.913406:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534727296/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773663.913425:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773663.913428:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773663.913430:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534727296:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773663.913435:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534727424:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773663.913451:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773663.913460:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773663/real 1790773663] req@ffff9c4ef73c2d80 x1877759534727424/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773663.913468:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534727424/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773663.913475:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773663.913477:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773663.913480:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534727424:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773663.913484:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534727552:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773663.913495:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773663.913501:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773663/real 1790773663] req@ffff9c4ef73c3800 x1877759534727552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773663.913509:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534727552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773663.913520:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773663.913522:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773663.913525:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534727552:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773663.913541:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534727680:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773663.913552:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773663.913560:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773663/real 1790773663] req@ffff9c4ef73c0a80 x1877759534727680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773663.913573:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534727680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773663.913582:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773663.913586:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773663.913590:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534727680:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773663.913595:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534727808:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773663.913610:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773663.913618:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773663/real 1790773663] req@ffff9c4ef73c0e00 x1877759534727808/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773663.913626:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534727808/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773663.913633:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773663.913636:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773663.913638:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534727808:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773663.913642:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534727936:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773663.913671:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773663.913679:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773663/real 1790773663] req@ffff9c4ef73c2300 x1877759534727936/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773663.913688:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534727936/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773663.913695:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773663.913697:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773663.913699:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534727936:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773663.976740:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773663.976740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773663.976740:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773663.976741:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773664.040722:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773664.040746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773664.040783:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773664.168722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773664.168728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773664.232756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773664.296723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773664.360754:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773664.360755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773664.360755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773664.424726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773664.488718:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773664.488725:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773664.488770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773664.616761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773664.616761:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773664.616761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773664.616761:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773664.680726:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773664.680738:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773664.680738:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773664.680742:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773664.680772:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773664.680772:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773664.808721:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773664.872719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773664.936775:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773665.000798:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773665.000800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773665.000801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773665.000802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773665.064752:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773665.064790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773665.064865:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773665.192725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773665.192730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773665.256716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773665.320710:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773665.384736:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773665.384770:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773665.384770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773665.448749:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773665.512790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773665.512796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773665.512800:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773665.640719:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773665.640723:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773665.640756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773665.640757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773665.704753:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773665.704753:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773665.704753:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773665.704754:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773665.704788:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773665.704790:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773665.832749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773665.896723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773665.960744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773666.024731:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773666.024731:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773666.024733:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773666.024740:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773666.088755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773666.088772:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773666.088790:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773666.216755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773666.216755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773666.280731:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773666.344769:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773666.408757:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773666.408757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773666.408757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773666.472735:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773666.536790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773666.536800:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773666.536811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773666.664723:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773666.664727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773666.664731:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773666.664750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773666.728752:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773666.728763:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773666.728763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773666.728804:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773666.728809:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773666.728814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773666.856775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773666.920770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773666.984753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773667.048773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773667.048774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773667.048774:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773667.048784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773667.112733:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773667.112761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773667.112922:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773667.240738:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773667.240746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773667.304761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773667.368730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773667.432761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773667.432761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773667.432764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773667.496738:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773667.560764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773667.560764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773667.560764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773667.688718:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773667.688737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773667.688743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773667.688770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773667.752743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773667.752754:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773667.752769:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773667.752775:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773667.752814:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773667.752834:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773667.880793:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773667.944734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773668.008752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773668.072743:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773668.072756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773668.072760:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773668.072809:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773668.136753:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773668.136754:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773668.136899:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773668.264732:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773668.264732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773668.328741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773668.392738:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773668.456775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773668.456791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773668.456792:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773668.520764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773668.584765:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773668.584765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773668.584771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773668.712764:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773668.712764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773668.712764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773668.712776:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773668.776742:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773668.776742:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773668.776753:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773668.776769:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773668.776777:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773668.776800:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773668.904756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773668.968760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773669.032790:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773669.032794:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773669.032800:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773669.032805:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773669.032814:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2664 00000100:00080000:2.0:1790773669.032823:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773669.032833:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773669.032867:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534728064/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773669.032889:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773669.032892:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773669.032893:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773669.032896:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2664 00000100:00080000:2.0:1790773669.032900:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773669.032904:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773669.032912:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773669.032922:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534728064:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773669.032924:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534728192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773669.032936:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773669.032939:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773669.032940:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773669.032944:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2664 00000100:00080000:2.0:1790773669.032946:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773669.032949:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773669.032952:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773669.032961:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534728320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773669.032968:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773669.032970:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773669.032972:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773669.032975:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2664 00000100:00080000:2.0:1790773669.032977:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773669.032980:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773669.032985:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790773669.032985:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773669.032996:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534728448/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773669.033008:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773669.033010:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773669.033012:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773669.033014:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2664 00000100:00080000:2.0:1790773669.033017:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773669.033021:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773669.033038:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773669.033042:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773669/real 1790773669] req@ffff9c4efbcc0a80 x1877759534728064/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773669.033053:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534728576/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773669.033062:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534728064/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773669.033062:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773669.033070:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773669.033073:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773669.033081:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2664 00000100:00080000:0.0:1790773669.033084:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773669.033085:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773669.033092:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773669.033095:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773669.033098:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773669.033108:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534728704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773669.033114:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534728064:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773669.033118:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773669.033120:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773669.033124:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773669.033131:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2664 00000100:00080000:2.0:1790773669.033135:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773669.033138:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773669.033142:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773669.033148:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534728832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773669.033156:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773669.033157:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534728192:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773669.033159:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773669.033162:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773669.033169:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2664 00000100:00080000:2.0:1790773669.033171:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773669.033175:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773669.033178:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773669.033190:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534728960/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773669.033196:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773669.033207:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773669.033211:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1790773669.033214:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773669/real 1790773669] req@ffff9c4efbcc2a00 x1877759534728192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773669.033215:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773669.033226:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2664 00000100:00100000:0.0:1790773669.033232:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534728192/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773669.033245:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773669.033247:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773669.033253:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773669.033254:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773669.033260:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534728192:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790773669.033262:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773669.033267:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534728320:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773669.033279:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534729088/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773669.033288:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773669.033298:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773669/real 1790773669] req@ffff9c4efbcc0380 x1877759534728320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773669.033313:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534728320/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773669.033320:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773669.033323:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773669.033327:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534728320:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773669.033331:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534728448:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773669.033342:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773669.033349:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773669/real 1790773669] req@ffff9c4efbcc1f80 x1877759534728448/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773669.033358:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534728448/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773669.033367:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773669.033370:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773669.033374:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534728448:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773669.033381:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534728576:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773669.033399:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773669.033412:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773669/real 1790773669] req@ffff9c4efbcc1c00 x1877759534728576/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773669.033422:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534728576/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773669.033429:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773669.033432:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773669.033434:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534728576:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773669.033439:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534728704:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773669.033453:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773669.033460:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773669/real 1790773669] req@ffff9c4efbcc3100 x1877759534728704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773669.033468:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534728704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773669.033483:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773669.033486:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773669.033488:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534728704:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773669.033504:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534728832:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773669.033515:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773669.033521:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773669/real 1790773669] req@ffff9c4efbcc2680 x1877759534728832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773669.033529:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534728832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773669.033535:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773669.033538:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773669.033540:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534728832:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773669.033544:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534728960:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773669.033553:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773669.033559:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773669/real 1790773669] req@ffff9c4efbcc0700 x1877759534728960/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773669.033567:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534728960/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773669.033574:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773669.033576:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773669.033579:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534728960:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773669.033582:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534729088:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773669.033592:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773669.033597:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773669/real 1790773669] req@ffff9c4efbcc1500 x1877759534729088/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773669.033605:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534729088/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773669.033611:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773669.033613:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773669.033616:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534729088:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773669.096742:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773669.096771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773669.096800:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773669.096806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773669.160764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773669.160834:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773669.160911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773669.288729:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773669.288729:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773669.352766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773669.416733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773669.480732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773669.480735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773669.480761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773669.544711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773669.608731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773669.608731:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773669.608764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773669.736745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773669.736751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773669.736766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773669.736769:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773669.800770:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773669.800777:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773669.800781:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773669.800804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773669.800812:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773669.800829:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773669.928706:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773669.992751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773670.056760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773670.120751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773670.120752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773670.120752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773670.120792:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773670.184776:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773670.184848:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773670.184863:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773670.312746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773670.312747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773670.376720:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773670.440729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773670.504743:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773670.504752:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773670.504752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773670.568770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773670.632745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773670.632753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773670.632767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773670.760770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773670.760771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773670.760771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773670.760797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773670.824720:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773670.824728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773670.824776:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773670.824780:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773670.824799:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773670.824811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773670.952766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773671.016743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773671.080752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773671.144735:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773671.144735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773671.144775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773671.144775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773671.208776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773671.208821:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773671.208840:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773671.336781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773671.336785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773671.400716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773671.464775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773671.528742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773671.528742:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773671.528759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773671.592711:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773671.656772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773671.656772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773671.656779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773671.784796:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773671.784797:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773671.784797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773671.784797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773671.848774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773671.848780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773671.848806:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773671.848807:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773671.848845:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773671.848862:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773671.976762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773672.040770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773672.104776:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773672.168812:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773672.168823:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773672.168824:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773672.168829:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773672.232758:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773672.232793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773672.232919:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773672.360719:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773672.360720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773672.424719:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773672.488763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773672.552813:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773672.552813:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773672.552817:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773672.616718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773672.680783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773672.680785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773672.680800:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773672.808829:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773672.808833:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773672.808836:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773672.808836:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773672.872721:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773672.872743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773672.872767:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773672.872767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773672.872773:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773672.872787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773673.000754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773673.064786:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773673.128721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773673.192717:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773673.192734:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773673.192738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773673.192790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773673.256735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773673.256760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773673.256836:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773673.384760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773673.384764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773673.448744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773673.512726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773673.576728:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773673.576728:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773673.576734:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773673.640726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773673.704745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773673.704745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773673.704749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773673.832740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773673.832741:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773673.832741:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773673.832743:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773673.896731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773673.896733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773673.896763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773673.896766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773673.896782:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773673.896799:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773674.024786:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773674.088752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773674.152720:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773674.152720:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773674.152730:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773674.152735:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773674.152743:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2670 00000100:00080000:3.0:1790773674.152752:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773674.152762:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773674.152808:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534729216/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773674.152840:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773674.152843:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773674.152844:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773674.152848:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2670 00000100:00080000:3.0:1790773674.152853:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773674.152857:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773674.152866:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773674.152871:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534729216:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773674.152897:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534729344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773674.152911:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773674.152915:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773674.152918:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773674.152923:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2670 00000100:00080000:3.0:1790773674.152943:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773674.152950:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773674.152958:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773674.152962:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773674.152978:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534729472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773674.152991:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773674.152995:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773674.152998:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773674.153003:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773674/real 1790773674] req@ffff9c4ef73c1c00 x1877759534729216/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773674.153004:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2670 00000100:00080000:3.0:1790773674.153011:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773674.153019:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773674.153028:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773674.153038:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534729216/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773674.153042:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534729600/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773674.153063:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773674.153067:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773674.153072:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773674.153073:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534729216:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773674.153080:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773674.153084:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773674.153088:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2670 00000100:00080000:3.0:1790773674.153091:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773674.153094:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773674.153099:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773674.153107:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534729344:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773674.153109:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534729728/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773674.153121:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773674.153124:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773674.153126:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773674.153130:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2670 00000100:00080000:3.0:1790773674.153132:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773674.153135:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773674.153140:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773674.153144:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773674.153153:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534729856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773674.153159:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773674/real 1790773674] req@ffff9c4ef73c2300 x1877759534729344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773674.153189:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534729344/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773674.153195:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773674.153202:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773674.153205:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773674.153211:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2670 00000100:00080000:3.0:1790773674.153215:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773674.153219:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773674.153220:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773674.153228:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773674.153229:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773674.153235:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534729344:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773674.153240:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534729984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773674.153243:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534729472:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773674.153256:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773674.153259:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773674.153261:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773674.153266:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2670 00000800:00000100:0.0:1790773674.153270:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773674.153272:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773674.153278:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773674.153283:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773674.153284:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773674/real 1790773674] req@ffff9c4ef73c0e00 x1877759534729472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773674.153294:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534730112/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773674.153299:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534729472/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773674.153307:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773674.153314:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773674.153317:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773674.153326:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2670 00000100:00080000:0.0:1790773674.153330:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773674.153331:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773674.153338:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773674.153339:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773674.153344:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773674.153345:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534729472:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773674.153352:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534729600:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773674.153359:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534730240/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773674.153370:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773674.153379:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773674/real 1790773674] req@ffff9c4ef73c0a80 x1877759534729600/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773674.153387:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534729600/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773674.153395:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773674.153398:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773674.153400:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534729600:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773674.153411:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534729728:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773674.153422:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773674.153429:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773674/real 1790773674] req@ffff9c4ef73c3800 x1877759534729728/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773674.153437:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534729728/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773674.153443:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773674.153445:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773674.153449:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534729728:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773674.153453:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534729856:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773674.153463:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773674.153468:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773674/real 1790773674] req@ffff9c4ef73c2d80 x1877759534729856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773674.153476:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534729856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773674.153487:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773674.153489:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773674.153492:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534729856:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773674.153496:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534729984:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773674.153505:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773674.153511:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773674/real 1790773674] req@ffff9c4ef73c2a00 x1877759534729984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773674.153519:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534729984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773674.153525:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773674.153527:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773674.153529:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534729984:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773674.153532:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534730112:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773674.153542:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773674.153549:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773674/real 1790773674] req@ffff9c4ef73c0700 x1877759534730112/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773674.153556:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534730112/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773674.153564:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773674.153565:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773674.153568:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534730112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773674.153571:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534730240:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773674.153581:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773674.153587:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773674/real 1790773674] req@ffff9c4ef73c1500 x1877759534730240/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773674.153595:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534730240/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773674.153601:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773674.153603:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773674.153605:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534730240:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773674.216765:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773674.216765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773674.216766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773674.216766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773674.280743:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773674.280767:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773674.280802:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773674.408715:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773674.408725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773674.472801:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773674.536752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773674.600760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773674.600762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773674.600791:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773674.664737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773674.728747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773674.728747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773674.728753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773674.856743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773674.856743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773674.856743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773674.856744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773674.920727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773674.920735:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773674.920735:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773674.920738:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773674.920768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773674.920768:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773675.048744:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773675.112749:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773675.176729:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773675.240776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773675.240814:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773675.240821:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773675.240821:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773675.304749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773675.304760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773675.304895:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773675.432740:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773675.432740:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773675.496757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773675.560748:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773675.624749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773675.624749:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773675.624749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773675.688713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773675.752769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773675.752777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773675.752796:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773675.880736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773675.880736:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773675.880736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773675.880769:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773675.944724:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773675.944724:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773675.944734:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773675.944746:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773675.944768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773675.944774:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773676.072775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773676.136729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773676.200741:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773676.264744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773676.264744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773676.264744:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773676.264744:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773676.328730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773676.328770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773676.328793:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773676.456721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773676.456726:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773676.520773:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773676.584762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773676.648735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773676.648735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773676.648736:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773676.712773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773676.776718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773676.776723:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773676.776724:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773676.904747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773676.904747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773676.904747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773676.904753:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773676.968722:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773676.968724:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773676.968734:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773676.968737:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773676.968762:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773676.968768:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773677.096733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773677.160752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773677.224753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773677.288818:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773677.288818:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773677.288820:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773677.288825:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773677.352758:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773677.352776:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773677.352886:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773677.480782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773677.480788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773677.544767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773677.608770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773677.672724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773677.672724:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773677.672729:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773677.736762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773677.800743:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773677.800743:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773677.800744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773677.928727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773677.928727:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773677.928727:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773677.928731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773677.992725:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773677.992752:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773677.992764:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773677.992768:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773677.992769:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773677.992796:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773678.120768:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773678.184757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773678.248727:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773678.312759:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773678.312760:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773678.312760:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773678.312760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773678.376718:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773678.376724:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773678.376916:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773678.504754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773678.504754:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773678.568779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773678.632752:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773678.696742:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773678.696748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773678.696792:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773678.760822:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773678.824736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773678.824736:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773678.824742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773678.952757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773678.952757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773678.952758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773678.952758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773679.016725:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773679.016726:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773679.016729:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773679.016732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773679.016760:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773679.016761:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773679.144774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773679.208760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773679.272768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773679.272769:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773679.272785:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773679.272790:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773679.272801:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2675 00000100:00080000:3.0:1790773679.272825:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773679.272851:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773679.272890:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534730368/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773679.272915:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773679.272919:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773679.272920:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773679.272924:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2675 00000100:00080000:3.0:1790773679.272927:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773679.272932:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773679.272939:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773679.272951:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534730496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773679.272957:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534730368:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773679.272980:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773679.272983:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773679.272984:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773679.272989:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2675 00000100:00080000:3.0:1790773679.272993:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773679.273000:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773679.273005:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773679.273014:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534730624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773679.273021:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773679.273026:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773679.273027:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773679.273030:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2675 00000100:00080000:3.0:1790773679.273033:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773679.273036:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773679.273040:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773679.273048:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534730752/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773679.273062:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773679.273065:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1790773679.273066:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773679.273068:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773679.273077:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2675 00000100:00080000:3.0:1790773679.273084:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773679.273094:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773679.273100:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773679.273117:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534730880/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773679.273124:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773679/real 1790773679] req@ffff9c4ef73c0000 x1877759534730368/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773679.273150:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534730368/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773679.273152:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773679.273165:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773679.273173:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773679.273181:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2675 00000100:00080000:3.0:1790773679.273184:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773679.273194:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773679.273198:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773679.273201:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773679.273211:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790773679.273216:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534731008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773679.273224:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534730368:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773679.273228:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773679.273237:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773679.273239:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773679.273245:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2675 00000100:00080000:3.0:1790773679.273248:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773679.273252:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773679.273256:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773679.273267:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534730496:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773679.273269:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534731136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773679.273281:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773679.273290:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773679.273291:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773679.273296:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2675 00000100:00080000:3.0:1790773679.273300:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773679.273307:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773679.273311:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773679.273324:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534731264/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773679.273326:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773679.273341:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773679.273344:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773679.273345:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773679.273351:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2675 00000100:00100000:0.0:1790773679.273353:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773679/real 1790773679] req@ffff9c4ef73c1500 x1877759534730496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773679.273356:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773679.273363:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773679.273367:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773679.273384:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534731392/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773679.273391:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534730496/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773679.273429:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773679.273437:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773679.273446:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534730496:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773679.273456:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534730624:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773679.273490:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773679.273504:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773679/real 1790773679] req@ffff9c4ef73c0700 x1877759534730624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773679.273519:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534730624/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773679.273530:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773679.273535:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773679.273539:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534730624:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773679.273547:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534730752:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773679.273571:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773679.273584:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773679/real 1790773679] req@ffff9c4ef73c2a00 x1877759534730752/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773679.273597:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534730752/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773679.273605:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773679.273609:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773679.273613:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534730752:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773679.273620:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534730880:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773679.273643:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773679.273682:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773679/real 1790773679] req@ffff9c4ef73c2d80 x1877759534730880/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773679.273696:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534730880/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773679.273707:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773679.273711:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773679.273715:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534730880:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773679.273723:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534731008:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773679.273745:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773679.273759:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773679/real 1790773679] req@ffff9c4ef73c3800 x1877759534731008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773679.273768:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534731008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773679.273781:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773679.273784:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773679.273787:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534731008:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773679.273806:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534731136:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773679.273821:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773679.273851:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773679/real 1790773679] req@ffff9c4ef73c0a80 x1877759534731136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773679.273861:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534731136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773679.273867:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773679.273870:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773679.273873:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534731136:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773679.273878:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534731264:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773679.273892:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773679.273899:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773679/real 1790773679] req@ffff9c4ef73c0e00 x1877759534731264/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773679.273907:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534731264/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773679.273914:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773679.273917:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773679.273919:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534731264:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773679.273923:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534731392:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773679.273934:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773679.273940:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773679/real 1790773679] req@ffff9c4ef73c2300 x1877759534731392/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773679.273947:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534731392/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773679.273954:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773679.273956:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773679.273958:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534731392:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773679.336746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773679.336746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773679.336749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773679.336806:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773679.400780:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773679.400800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773679.400844:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773679.528746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773679.528750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773679.592739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773679.656765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773679.720764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773679.720764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773679.720797:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773679.784787:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773679.848759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773679.848775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773679.848778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773679.976734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773679.976760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773679.976764:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773679.976856:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773680.040784:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773680.040801:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773680.040801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773680.040802:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773680.040847:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773680.040867:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773680.168745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773680.232723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773680.296726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773680.360766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773680.360766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773680.360767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773680.360779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773680.424757:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773680.424782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773680.424837:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773680.552784:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773680.552792:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773680.616788:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773680.680720:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773680.744743:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773680.744743:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773680.744776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773680.808721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773680.872763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773680.872772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773680.872772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773681.000711:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773681.000726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773681.000748:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773681.000754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773681.064745:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773681.064759:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773681.064760:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773681.064760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773681.064810:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773681.064814:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773681.192753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773681.256761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773681.320754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773681.384735:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773681.384761:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773681.384761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773681.384810:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773681.448754:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773681.448773:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773681.448865:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773681.576748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773681.576810:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773681.640840:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773681.704807:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773681.768812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773681.768812:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773681.768818:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773681.832737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773681.896744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773681.896793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773681.896806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773682.024765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773682.024765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773682.024798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773682.024798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773682.088790:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773682.088790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773682.088807:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773682.088807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773682.088835:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773682.088845:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773682.216754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773682.280760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773682.344739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773682.408781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773682.408782:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773682.408801:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773682.408809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773682.472796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773682.472796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773682.472883:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773682.600742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773682.600742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773682.664737:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773682.728727:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773682.792747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773682.792748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773682.792748:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773682.856740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773682.920779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773682.920780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773682.920798:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773683.048738:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773683.048770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773683.048770:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773683.048771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773683.112814:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773683.112815:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773683.112824:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773683.112831:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773683.112849:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773683.112869:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773683.240785:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773683.304776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773683.368725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773683.432767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773683.432771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773683.432811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773683.432816:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773683.496765:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773683.496809:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773683.496882:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773683.624780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773683.624780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773683.688778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773683.752727:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773683.816805:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773683.816812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773683.816812:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773683.880770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773683.944780:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773683.944796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773683.944803:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773684.072774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773684.072775:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773684.072780:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773684.072782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773684.136743:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773684.136767:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773684.136793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773684.136806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773684.136807:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773684.136830:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773684.264805:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773684.328753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773684.392764:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773684.392773:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773684.392777:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773684.392786:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2680 00000100:00080000:0.0:1790773684.392795:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773684.392809:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773684.392842:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534731520/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773684.392873:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773684.392876:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773684.392878:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773684.392882:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2680 00000100:00080000:0.0:1790773684.392885:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773684.392889:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773684.392897:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773684.392909:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534731648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773684.392918:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773684.392922:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773684.392924:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773684.392930:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2680 00000100:00080000:0.0:1790773684.392934:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773684.392944:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773684.392946:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534731520:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773684.392952:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773684.392983:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534731776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773684.392992:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773684.392996:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773684.392998:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773684.393002:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2680 00000100:00080000:0.0:1790773684.393006:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773684.393011:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773684.393016:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773684.393026:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534731904/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773684.393033:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773684.393036:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773684.393038:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773684.393041:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2680 00000100:00080000:0.0:1790773684.393045:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773684.393049:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773684.393053:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:1.0:1790773684.393063:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773684.393064:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534732032/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773684.393070:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773684.393083:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773684.393089:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773684.393092:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773684.393099:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2680 00000100:00080000:0.0:1790773684.393106:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773684.393113:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773684.393119:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773684.393130:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534732160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773684.393140:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773684.393140:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773684/real 1790773684] req@ffff9c4ec7a7d180 x1877759534731520/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773684.393154:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773684.393164:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773684.393174:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2680 00000100:00080000:0.0:1790773684.393182:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773684.393191:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773684.393196:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773684.393204:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534732288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773684.393211:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534731520/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773684.393249:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773684.393261:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773684.393263:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773684.393270:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2680 00000100:00080000:1.0:1790773684.393270:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773684.393278:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773684.393285:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773684.393289:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773684.393297:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534732416/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773684.393297:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773684.393316:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534731520:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773684.393320:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773684.393328:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773684.393331:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773684.393340:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2680 00000100:00100000:1.0:1790773684.393341:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534731648:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773684.393344:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773684.393353:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773684.393357:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773684.393376:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534732544/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773684.393398:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773684.393421:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773684/real 1790773684] req@ffff9c4ec7a7c700 x1877759534731648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773684.393439:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534731648/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773684.393468:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773684.393474:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773684.393479:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534731648:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773684.393512:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534731776:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773684.393552:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773684.393568:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773684/real 1790773684] req@ffff9c4ec7a7e300 x1877759534731776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773684.393583:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534731776/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773684.393595:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773684.393600:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773684.393604:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534731776:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773684.393612:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534731904:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773684.393637:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773684.393689:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773684/real 1790773684] req@ffff9c4ec7a7fb80 x1877759534731904/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773684.393707:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534731904/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773684.393741:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773684.393746:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773684.393752:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534731904:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773684.393760:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534732032:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773684.393787:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773684.393801:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773684/real 1790773684] req@ffff9c4ec7a7ca80 x1877759534732032/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773684.393817:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534732032/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773684.393829:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773684.393833:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773684.393838:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534732032:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773684.393845:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534732160:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773684.393871:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773684.393885:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773684/real 1790773684] req@ffff9c4ec7a7c380 x1877759534732160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773684.393900:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534732160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773684.393918:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773684.393923:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773684.393927:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534732160:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773684.393935:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534732288:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773684.393957:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773684.393972:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773684/real 1790773684] req@ffff9c4ec7a7d880 x1877759534732288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773684.393985:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534732288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773684.393996:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773684.394000:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773684.394004:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534732288:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773684.394012:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534732416:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773684.394034:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773684.394047:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773684/real 1790773684] req@ffff9c4ec7a7ed80 x1877759534732416/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773684.394062:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534732416/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773684.394076:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773684.394080:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773684.394085:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534732416:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773684.394093:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534732544:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773684.394116:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773684.394129:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773684/real 1790773684] req@ffff9c4ec7a7df80 x1877759534732544/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773684.394144:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534732544/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773684.394157:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773684.394161:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773684.394165:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534732544:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773684.456826:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773684.456826:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773684.456826:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773684.456828:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773684.520757:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773684.520800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773684.520889:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773684.648747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773684.648771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773684.712793:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773684.776781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773684.840812:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773684.840812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773684.840812:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773684.904792:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773684.968792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773684.968799:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773684.968811:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773685.096777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773685.096777:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773685.096785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773685.096785:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773685.160806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773685.160806:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773685.160820:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773685.160820:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773685.160856:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773685.160864:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773685.288825:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773685.352823:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773685.416801:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773685.480745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773685.480745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773685.480751:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773685.480790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773685.544745:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773685.544788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773685.544842:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773685.672764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773685.672780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773685.736771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773685.800794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773685.864789:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773685.864789:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773685.864789:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773685.928773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773685.992811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773685.992812:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773685.992822:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773686.120806:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773686.120806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773686.120813:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773686.120813:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773686.184741:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773686.184761:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773686.184789:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773686.184806:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773686.184807:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773686.184811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773686.312790:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773686.376771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773686.440713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773686.504754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773686.504754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773686.504756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773686.504782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773686.568759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773686.568759:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773686.568807:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773686.696745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773686.696747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773686.760765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773686.824779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773686.888775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773686.888775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773686.888775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773686.952750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773687.016784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773687.016784:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773687.016790:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773687.144765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773687.144765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773687.144765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773687.144767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773687.208747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773687.208750:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773687.208751:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773687.208752:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773687.208784:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773687.208785:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773687.336766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773687.400727:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773687.464748:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773687.528748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773687.528750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773687.528881:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773687.528897:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773687.592735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773687.592788:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773687.592853:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773687.720736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773687.720736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773687.784771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773687.848756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773687.912790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773687.912791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773687.912794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773687.976772:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773688.040779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773688.040779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773688.040779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773688.168771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773688.168771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773688.168771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773688.168772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773688.232746:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773688.232749:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773688.232753:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773688.232771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773688.232793:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773688.232796:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773688.360788:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773688.424763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773688.488772:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773688.552743:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773688.552743:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773688.552771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773688.552791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773688.616774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773688.616794:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773688.616825:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773688.744768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773688.744768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773688.808766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773688.872723:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773688.936758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773688.936758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773688.936758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773689.000718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773689.064773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773689.064773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773689.064773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773689.192763:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773689.192763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773689.192763:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773689.192789:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773689.256740:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773689.256741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773689.256741:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773689.256745:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773689.256774:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773689.256787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773689.384707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773689.448713:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773689.512715:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773689.512722:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773689.512728:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773689.512732:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773689.512740:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2685 00000100:00080000:0.0:1790773689.512749:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773689.512759:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773689.512794:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534732672/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773689.512825:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773689.512829:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773689.512831:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773689.512836:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2685 00000100:00080000:0.0:1790773689.512840:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773689.512845:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773689.512854:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773689.512868:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534732800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773689.512879:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534732672:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773689.512883:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773689.512889:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773689.512891:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773689.512896:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2685 00000100:00080000:0.0:1790773689.512901:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773689.512907:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773689.512913:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773689.512927:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534732928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773689.512935:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773689.512938:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773689.512939:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773689.512944:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2685 00000800:00000100:1.0:1790773689.512946:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773689.512949:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773689.512957:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773689.512963:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773689.512978:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534733056/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773689.512990:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773689/real 1790773689] req@ffff9c4ec7a7f480 x1877759534732672/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773689.512991:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773689.513001:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773689.513006:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773689.513013:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2685 00000100:00080000:0.0:1790773689.513017:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773689.513020:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534732672/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773689.513024:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773689.513032:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:1.0:1790773689.513041:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773689.513046:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534733184/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773689.513050:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773689.513060:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534732672:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773689.513062:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773689.513070:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773689.513072:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773689.513076:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2685 00000100:00080000:0.0:1790773689.513079:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773689.513082:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773689.513085:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773689.513092:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534732800:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773689.513095:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534733312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773689.513106:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773689.513109:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773689.513111:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773689.513115:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2685 00000100:00080000:0.0:1790773689.513117:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773689.513123:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773689.513124:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773689.513127:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773689.513137:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534733440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773689.513143:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773689/real 1790773689] req@ffff9c4ec7a7df80 x1877759534732800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773689.513148:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773689.513155:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773689.513158:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773689.513164:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2685 00000100:00080000:0.0:1790773689.513168:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773689.513174:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773689.513175:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534732800/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773689.513177:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773689.513188:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534733568/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773689.513197:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773689.513200:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773689.513203:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773689.513204:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773689.513211:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2685 00000100:00080000:0.0:1790773689.513215:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773689.513215:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773689.513223:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773689.513226:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534732800:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773689.513228:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773689.513239:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534733696/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773689.513242:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534732928:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773689.513270:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773689.513281:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773689/real 1790773689] req@ffff9c4ec7a7ed80 x1877759534732928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773689.513295:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534732928/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773689.513305:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773689.513308:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773689.513312:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534732928:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773689.513319:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534733056:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773689.513336:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773689.513345:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773689/real 1790773689] req@ffff9c4ec7a7d880 x1877759534733056/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773689.513358:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534733056/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773689.513368:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773689.513372:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773689.513376:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534733056:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773689.513382:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534733184:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773689.513398:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773689.513407:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773689/real 1790773689] req@ffff9c4ec7a7c380 x1877759534733184/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773689.513419:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534733184/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773689.513436:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773689.513440:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773689.513443:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534733184:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773689.513459:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534733312:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773689.513475:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773689.513485:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773689/real 1790773689] req@ffff9c4ec7a7ca80 x1877759534733312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773689.513498:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534733312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773689.513514:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773689.513517:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773689.513521:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534733312:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773689.513527:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534733440:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773689.513543:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773689.513552:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773689/real 1790773689] req@ffff9c4ec7a7fb80 x1877759534733440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773689.513565:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534733440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773689.513573:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773689.513577:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773689.513582:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534733440:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773689.513588:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534733568:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773689.513604:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773689.513613:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773689/real 1790773689] req@ffff9c4ec7a7e300 x1877759534733568/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773689.513625:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534733568/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773689.513672:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773689.513677:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773689.513681:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534733568:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773689.513687:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534733696:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773689.513708:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773689.513719:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773689/real 1790773689] req@ffff9c4ec7a7c700 x1877759534733696/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773689.513732:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534733696/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773689.513742:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773689.513746:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773689.513750:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534733696:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773689.576757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773689.576758:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773689.576759:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773689.576759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773689.640737:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773689.640737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773689.640782:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773689.768751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773689.768751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773689.832740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773689.896747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773689.960767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773689.960767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773689.960768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773690.024721:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773690.088744:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773690.088744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773690.088744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773690.216757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773690.216757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773690.216783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773690.216790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773690.280713:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773690.280758:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773690.280760:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773690.280796:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773690.280812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773690.280815:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773690.408800:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773690.472777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773690.536772:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773690.600737:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773690.600766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773690.600766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773690.600879:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773690.664728:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773690.664789:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773690.664913:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773690.792732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773690.792786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773690.856727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773690.920733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773690.984703:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773690.984713:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773690.984781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773691.048741:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773691.112748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773691.112748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773691.112748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773691.240738:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773691.240744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773691.240755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773691.240770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773691.304755:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773691.304779:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773691.304781:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773691.304785:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773691.304805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773691.304829:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773691.432753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773691.496734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773691.560763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773691.624767:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773691.624767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773691.624768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773691.624772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773691.688786:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773691.688796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773691.688909:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773691.816751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773691.816777:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773691.880732:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773691.944776:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773692.008702:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773692.008727:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773692.008769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773692.072735:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773692.136715:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773692.136726:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773692.136733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773692.264744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773692.264744:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773692.264744:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773692.264756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773692.328749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773692.328749:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773692.328769:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773692.328797:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773692.328822:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773692.328839:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773692.456718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773692.520753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773692.584726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773692.648751:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773692.648759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773692.648761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773692.648762:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773692.712770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773692.712791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773692.712905:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773692.840737:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773692.840738:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773692.904764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773692.968743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773693.032715:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773693.032731:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773693.032759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773693.096761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773693.160752:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773693.160752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773693.160753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773693.288755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773693.288755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773693.288755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773693.288760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773693.352752:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773693.352772:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773693.352776:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773693.352783:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773693.352799:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773693.352807:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773693.480753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773693.544755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773693.608725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773693.672727:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773693.672727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773693.672731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773693.672772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773693.736760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773693.736838:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773693.736897:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773693.864728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773693.864728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773693.928703:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773693.992728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773694.056732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773694.056776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773694.056789:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773694.120744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773694.184732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773694.184733:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773694.184757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773694.312769:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773694.312769:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773694.312769:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773694.312777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773694.376741:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773694.376741:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773694.376760:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773694.376781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773694.376794:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773694.376815:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773694.504729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773694.568752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773694.632739:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773694.632749:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773694.632754:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773694.632763:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2690 00000100:00080000:0.0:1790773694.632773:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773694.632786:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773694.632797:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773694.632842:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534733824/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773694.632865:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773694.632868:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773694.632870:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773694.632874:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2690 00000100:00080000:0.0:1790773694.632878:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773694.632883:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773694.632891:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773694.632904:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534733952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773694.632917:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534733824:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773694.632921:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773694.632932:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773694.632934:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773694.632938:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2690 00000100:00080000:0.0:1790773694.632941:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773694.632944:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773694.632948:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773694.632959:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534734080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773694.632967:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773694.632970:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773694.632971:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773694.632975:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2690 00000100:00080000:0.0:1790773694.632978:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773694.632981:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773694.632985:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773694.632998:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534734208/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773694.633004:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773694.633007:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773694.633009:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773694.633013:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2690 00000100:00080000:0.0:1790773694.633016:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773694.633020:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773694.633025:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:1.0:1790773694.633031:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773694.633036:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534734336/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773694.633045:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773694.633049:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773694.633052:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773694.633056:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2690 00000100:00080000:0.0:1790773694.633060:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773694.633070:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773694.633076:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773694.633090:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534734464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773694.633103:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773694.633103:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773694/real 1790773694] req@ffff9c4ec7a7d180 x1877759534733824/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773694.633120:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773694.633126:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773694.633138:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2690 00000100:00080000:0.0:1790773694.633145:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773694.633155:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534733824/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773694.633156:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773694.633170:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773694.633191:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773694.633192:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534734592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773694.633208:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773694.633226:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534733824:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773694.633228:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773694.633240:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773694.633244:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773694.633250:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2690 00000100:00080000:0.0:1790773694.633253:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773694.633257:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773694.633261:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773694.633270:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534734720/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773694.633280:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534733952:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773694.633282:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773694.633296:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773694.633307:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773694.633314:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2690 00000100:00080000:0.0:1790773694.633324:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773694.633333:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773694.633336:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773694.633339:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773694.633353:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534734848/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773694.633370:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773694/real 1790773694] req@ffff9c4ec7a7c700 x1877759534733952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773694.633388:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534733952/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773694.633409:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773694.633414:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773694.633418:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534733952:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773694.633426:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534734080:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773694.633452:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773694.633466:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773694/real 1790773694] req@ffff9c4ec7a7e300 x1877759534734080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773694.633481:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534734080/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773694.633491:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773694.633496:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773694.633500:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534734080:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773694.633509:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534734208:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773694.633554:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773694.633569:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773694/real 1790773694] req@ffff9c4ec7a7fb80 x1877759534734208/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773694.633584:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534734208/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773694.633595:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773694.633600:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773694.633604:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534734208:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773694.633611:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534734336:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773694.633635:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773694.633648:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773694/real 1790773694] req@ffff9c4ec7a7ca80 x1877759534734336/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773694.633703:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534734336/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773694.633715:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773694.633720:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773694.633724:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534734336:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773694.633732:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534734464:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773694.633756:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773694.633769:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773694/real 1790773694] req@ffff9c4ec7a7c380 x1877759534734464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773694.633783:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534734464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773694.633801:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773694.633805:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773694.633809:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534734464:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773694.633817:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534734592:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773694.633839:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773694.633853:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773694/real 1790773694] req@ffff9c4ec7a7d880 x1877759534734592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773694.633867:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534734592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773694.633876:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773694.633880:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773694.633884:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534734592:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773694.633908:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534734720:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773694.633930:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773694.633941:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773694/real 1790773694] req@ffff9c4ec7a7ed80 x1877759534734720/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773694.633955:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534734720/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773694.633973:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773694.633978:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773694.633981:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534734720:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773694.633988:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534734848:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773694.634006:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773694.634016:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773694/real 1790773694] req@ffff9c4ec7a7df80 x1877759534734848/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773694.634029:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534734848/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773694.634039:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773694.634043:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773694.634046:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534734848:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773694.696712:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773694.696735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773694.696768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773694.696769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773694.760794:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773694.760814:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773694.760909:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773694.888752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773694.888768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773694.952782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773695.016761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773695.080728:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773695.080743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773695.080760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773695.144789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773695.208759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773695.208764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773695.208794:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773695.336789:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773695.336795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773695.336813:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773695.336819:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773695.400767:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773695.400768:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773695.400778:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773695.400780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773695.400810:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773695.400822:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773695.528764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773695.592750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773695.656774:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773695.720807:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773695.720807:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773695.720819:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773695.720830:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773695.784748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773695.784790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773695.784807:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773695.912767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773695.912808:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773695.976807:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773696.041025:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773696.104708:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773696.104716:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773696.104760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773696.168758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773696.232768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773696.232768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773696.232792:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773696.360835:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773696.360857:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773696.360861:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773696.360862:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773696.424771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773696.424784:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773696.424784:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773696.424792:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773696.424833:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773696.424840:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773696.552726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773696.616769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773696.680733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773696.744849:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773696.744850:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773696.744852:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773696.744852:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773696.808798:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773696.808801:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773696.808913:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773696.936764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773696.936773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773697.000715:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773697.064775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773697.128787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773697.128798:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773697.128810:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773697.192761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773697.256745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773697.256757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773697.256757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773697.384756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773697.384763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773697.384788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773697.384788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773697.448750:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773697.448767:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773697.448792:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773697.448792:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773697.448815:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773697.448836:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773697.576753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773697.640786:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773697.704729:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773697.768768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773697.768788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773697.768799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773697.768801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773697.832760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773697.832786:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773697.832943:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773697.960794:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773697.960799:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773698.024801:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773698.088753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773698.152751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773698.152761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773698.152767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773698.216789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773698.280811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773698.280813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773698.280813:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773698.408769:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773698.408780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773698.408820:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773698.408821:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773698.472760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773698.472780:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773698.472781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773698.472790:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773698.472851:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773698.472852:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773698.600769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773698.664743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773698.728725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773698.792761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773698.792761:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773698.792773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773698.792773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773698.856819:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773698.856820:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773698.856950:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773698.984751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773698.984760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773699.048804:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773699.112761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773699.176766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773699.176766:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773699.176773:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773699.240725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773699.304722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773699.304723:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773699.304826:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773699.432725:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773699.432727:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773699.432732:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773699.432773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773699.496784:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773699.496788:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773699.496788:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773699.496788:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773699.496860:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773699.496862:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773699.624746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773699.688734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773699.752761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773699.752786:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773699.752798:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773699.752803:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773699.752815:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2695 00000100:00080000:2.0:1790773699.752826:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773699.752836:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773699.752874:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534734976/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773699.752914:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773699.752919:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773699.752922:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773699.752927:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2695 00000100:00080000:2.0:1790773699.752932:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773699.752940:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773699.752951:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773699.752967:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534735104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773699.752975:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534734976:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773699.752985:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773699.752989:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773699.752991:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773699.752995:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2695 00000100:00080000:2.0:1790773699.752998:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773699.753003:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773699.753007:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773699.753017:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534735232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773699.753023:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773699.753026:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773699.753027:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773699.753031:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2695 00000100:00080000:2.0:1790773699.753033:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773699.753036:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773699.753040:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773699.753048:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534735360/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773699.753061:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773699.753064:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773699.753065:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773699.753072:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2695 00000800:00000100:1.0:1790773699.753083:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773699.753083:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773699.753092:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773699.753097:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773699.753104:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534735488/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773699.753112:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773699.753115:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773699.753117:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773699.753121:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2695 00000100:00080000:2.0:1790773699.753123:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773699.753127:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773699.753131:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773699.753140:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534735616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773699.753148:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773699.753151:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773699.753156:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773699.753157:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773699/real 1790773699] req@ffff9c4efbcc2300 x1877759534734976/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773699.753169:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2695 00000100:00080000:2.0:1790773699.753177:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773699.753185:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773699.753195:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773699.753209:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534735744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773699.753217:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773699.753220:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773699.753221:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773699.753227:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2695 00000100:00080000:2.0:1790773699.753229:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773699.753232:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773699.753236:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534734976/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773699.753237:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773699.753259:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534735872/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773699.753260:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773699.753277:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773699.753278:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773699.753287:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773699.753290:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773699.753294:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534734976:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773699.753298:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2695 00000100:00080000:2.0:1790773699.753306:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773699.753313:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773699.753317:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773699.753325:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534736000/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773699.753346:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534735104:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773699.753373:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773699.753384:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773699/real 1790773699] req@ffff9c4efbcc1880 x1877759534735104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773699.753395:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534735104/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773699.753411:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773699.753413:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773699.753416:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534735104:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773699.753421:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534735232:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773699.753455:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773699.753464:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773699/real 1790773699] req@ffff9c4efbcc1500 x1877759534735232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773699.753473:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534735232/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773699.753486:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773699.753489:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773699.753491:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534735232:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773699.753497:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534735360:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773699.753510:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773699.753517:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773699/real 1790773699] req@ffff9c4efbcc0700 x1877759534735360/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773699.753525:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534735360/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773699.753532:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773699.753535:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773699.753537:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534735360:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773699.753542:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534735488:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773699.753553:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773699.753560:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773699/real 1790773699] req@ffff9c4efbcc2680 x1877759534735488/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773699.753568:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534735488/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773699.753575:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773699.753577:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773699.753580:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534735488:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773699.753585:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534735616:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773699.753596:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773699.753602:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773699/real 1790773699] req@ffff9c4efbcc3100 x1877759534735616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773699.753610:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534735616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773699.753622:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773699.753624:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773699.753626:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534735616:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773699.753630:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534735744:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773699.753641:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773699.753647:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773699/real 1790773699] req@ffff9c4efbcc1c00 x1877759534735744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773699.753688:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534735744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773699.753698:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773699.753702:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773699.753706:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534735744:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773699.753714:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534735872:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773699.753733:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773699.753742:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773699/real 1790773699] req@ffff9c4efbcc1f80 x1877759534735872/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773699.753750:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534735872/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773699.753758:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773699.753760:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773699.753763:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534735872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773699.753767:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534736000:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773699.753779:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773699.753785:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773699/real 1790773699] req@ffff9c4efbcc0380 x1877759534736000/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773699.753793:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534736000/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773699.753799:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773699.753801:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773699.753804:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534736000:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773699.816753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773699.816754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773699.816755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773699.816762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773699.880717:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773699.880762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773699.880838:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773700.008733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773700.008733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773700.072782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773700.136748:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773700.200738:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773700.200738:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773700.200797:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773700.264745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773700.328764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773700.328764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773700.328765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773700.456837:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773700.456838:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773700.456843:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773700.456843:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773700.520765:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773700.520770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773700.520771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773700.520802:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773700.520805:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773700.520821:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773700.648713:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773700.712806:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773700.776735:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773700.840768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773700.840769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773700.840769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773700.840780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773700.904770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773700.904770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773700.904841:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773701.032752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773701.032752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773701.096739:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773701.160789:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773701.224751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773701.224752:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773701.224783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773701.288728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773701.352720:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773701.352724:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773701.352732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773701.480767:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773701.480767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773701.480781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773701.480782:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773701.544791:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773701.544806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773701.544808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773701.544849:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773701.544857:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773701.544871:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773701.673708:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773701.736778:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773701.800773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773701.864755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773701.864755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773701.864755:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773701.864797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773701.928791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773701.928792:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773701.928866:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773702.056723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773702.056736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773702.120770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773702.184712:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773702.248749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773702.248749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773702.248762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773702.312758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773702.376762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773702.376762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773702.376763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773702.504726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773702.504773:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773702.504773:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773702.504776:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773702.568768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773702.568769:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773702.568770:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773702.568771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773702.568816:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773702.568846:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773702.696762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773702.760721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773702.824761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773702.888838:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773702.888838:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773702.888839:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773702.888839:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773702.952752:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773702.952788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773702.952856:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773703.080726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773703.080762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773703.144754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773703.208798:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773703.272816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773703.272816:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773703.272817:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773703.336820:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773703.400735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773703.400735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773703.400786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773703.528785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773703.528786:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773703.528786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773703.528800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773703.592737:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773703.592738:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773703.592738:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773703.592746:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773703.592779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773703.592784:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773703.721698:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773703.784750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773703.848769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773703.912783:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773703.912783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773703.912793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773703.912795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773703.976779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773703.976805:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773703.976810:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773704.104721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773704.104773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773704.168766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773704.232765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773704.296842:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773704.296842:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773704.296843:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773704.360753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773704.424755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773704.424755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773704.424755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773704.552731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773704.552731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773704.552731:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773704.552736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773704.616741:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773704.616741:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773704.616741:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773704.616741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773704.616774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773704.616777:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773704.744723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773704.808731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773704.872737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773704.872738:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773704.872746:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773704.872751:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773704.872762:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2700 00000100:00080000:2.0:1790773704.872773:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773704.872786:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773704.872824:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534736128/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773704.872847:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773704.872851:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773704.872853:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773704.872858:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2700 00000100:00080000:2.0:1790773704.872862:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773704.872868:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773704.872878:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773704.872887:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534736128:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773704.872893:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534736256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773704.872904:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773704.872908:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773704.872910:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773704.872914:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2700 00000100:00080000:2.0:1790773704.872917:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773704.872922:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773704.872927:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773704.872937:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534736384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773704.872945:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773704.872946:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773704.872949:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773704.872951:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773704.872965:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2700 00000100:00080000:2.0:1790773704.872969:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773704.872974:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773704.872979:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773704.872987:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773704/real 1790773704] req@ffff9c4efbcc2a00 x1877759534736128/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773704.872991:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534736512/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773704.873001:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773704.873004:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773704.873007:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534736128/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773704.873008:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773704.873012:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2700 00000100:00080000:2.0:1790773704.873016:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773704.873018:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773704.873021:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773704.873023:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773704.873026:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773704.873027:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534736128:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773704.873036:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534736640/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773704.873045:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773704.873048:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773704.873051:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773704.873052:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534736256:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773704.873056:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2700 00000100:00080000:2.0:1790773704.873059:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773704.873064:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773704.873070:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773704.873074:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773704.873081:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534736768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773704.873089:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773704/real 1790773704] req@ffff9c4efbcc0a80 x1877759534736256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773704.873092:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773704.873097:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773704.873099:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773704.873103:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534736256/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773704.873104:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2700 00000100:00080000:2.0:1790773704.873108:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773704.873113:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773704.873117:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773704.873121:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773704.873125:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773704.873126:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534736896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773704.873129:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534736256:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773704.873135:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534736384:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773704.873136:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773704.873140:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773704.873142:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773704.873146:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2700 00000100:00080000:2.0:1790773704.873150:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773704.873153:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773704.873155:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773704.873160:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773704.873164:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773704/real 1790773704] req@ffff9c4efbcc0380 x1877759534736384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773704.873170:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534737024/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773704.873177:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534736384/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773704.873180:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773704.873184:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773704.873186:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773704.873191:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2700 00000100:00080000:1.0:1790773704.873194:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773704.873194:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773704.873198:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773704.873199:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773704.873202:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534736384:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790773704.873203:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773704.873209:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534736512:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773704.873212:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534737152/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773704.873227:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773704.873239:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773704/real 1790773704] req@ffff9c4efbcc1f80 x1877759534736512/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773704.873252:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534736512/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773704.873263:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773704.873266:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773704.873269:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534736512:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773704.873274:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534736640:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773704.873290:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773704.873298:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773704/real 1790773704] req@ffff9c4efbcc1c00 x1877759534736640/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773704.873306:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534736640/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773704.873312:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773704.873315:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773704.873320:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534736640:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773704.873355:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534736768:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773704.873378:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773704.873391:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773704/real 1790773704] req@ffff9c4efbcc3100 x1877759534736768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773704.873400:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534736768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773704.873412:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773704.873415:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773704.873417:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534736768:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773704.873422:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534736896:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773704.873436:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773704.873443:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773704/real 1790773704] req@ffff9c4efbcc2680 x1877759534736896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773704.873451:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534736896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773704.873457:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773704.873459:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773704.873461:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534736896:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773704.873464:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534737024:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773704.873474:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773704.873480:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773704/real 1790773704] req@ffff9c4efbcc0700 x1877759534737024/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773704.873492:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534737024/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773704.873503:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773704.873506:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773704.873510:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534737024:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773704.873516:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534737152:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773704.873537:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773704.873548:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773704/real 1790773704] req@ffff9c4efbcc1500 x1877759534737152/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773704.873557:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534737152/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773704.873564:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773704.873566:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773704.873569:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534737152:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773704.936714:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773704.936722:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773704.936722:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773704.936727:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773705.000735:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773705.000735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773705.000778:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773705.128752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773705.128752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773705.192752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773705.256739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773705.320750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773705.320750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773705.320750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773705.384698:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773705.448741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773705.448741:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773705.448745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773705.576754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773705.576755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773705.576755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773705.576755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773705.640741:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773705.640741:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773705.640743:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773705.640744:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773705.640775:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773705.640781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773705.768718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773705.832733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773705.896706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773705.960737:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773705.960737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773705.960738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773705.960751:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773706.024767:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773706.024777:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773706.024793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773706.152726:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773706.152728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773706.216766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773706.280729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773706.344742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773706.344742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773706.344742:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773706.408712:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773706.472760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773706.472760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773706.472760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773706.600721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773706.600724:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773706.600728:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773706.600736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773706.664750:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773706.664751:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773706.664751:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773706.664782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773706.664796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773706.664801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773706.792740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773706.856730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773706.920740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773706.984721:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773706.984722:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773706.984729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773706.984730:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773707.048745:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773707.048745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773707.048799:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773707.176710:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773707.176712:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773707.240761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773707.304727:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773707.368755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773707.368755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773707.368755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773707.432728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773707.496754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773707.496754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773707.496754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773707.624762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773707.624762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773707.624762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773707.624762:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773707.688714:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773707.688729:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773707.688747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773707.688758:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773707.688760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773707.688771:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773707.816728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773707.880762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773707.944784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773708.008753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773708.008754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773708.008754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773708.008783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773708.072805:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773708.072845:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773708.072883:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773708.200761:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773708.200761:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773708.264775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773708.328754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773708.392712:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773708.392759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773708.392759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773708.456713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773708.520767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773708.520767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773708.520782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773708.648754:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773708.648754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773708.648754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773708.648788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773708.712767:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773708.712778:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773708.712793:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773708.712797:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773708.712827:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773708.712836:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773708.840787:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773708.904723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773708.968739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773709.032739:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773709.032739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773709.032740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773709.032740:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773709.096735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773709.096753:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773709.096785:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773709.224751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773709.224751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773709.288749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773709.352734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773709.416738:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773709.416738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773709.416738:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773709.480713:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773709.544727:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773709.544727:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773709.544728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773709.672726:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773709.672727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773709.672728:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773709.672737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773709.736721:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773709.736721:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773709.736749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773709.736760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773709.736768:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773709.736788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773709.864717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773709.928767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773709.992770:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773709.992770:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773709.992784:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773709.992791:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773709.992805:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2705 00000100:00080000:1.0:1790773709.992819:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773709.992836:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773709.992901:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534737280/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773709.992943:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773709.992948:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773709.992950:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773709.992955:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2705 00000100:00080000:1.0:1790773709.992960:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773709.992966:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773709.992979:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773709.992997:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534737408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773709.993009:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534737280:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773709.993013:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773709.993021:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773709.993023:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773709.993030:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2705 00000100:00080000:1.0:1790773709.993034:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773709.993040:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773709.993047:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773709.993062:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534737536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773709.993074:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773709.993078:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773709.993082:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1790773709.993097:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773709.993098:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2705 00000100:00080000:1.0:1790773709.993105:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773709.993114:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773709.993121:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773709.993137:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534737664/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773709.993148:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773709/real 1790773709] req@ffff9c4efac4e300 x1877759534737280/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773709.993151:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773709.993161:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773709.993165:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773709.993173:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2705 00000100:00100000:0.0:1790773709.993177:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534737280/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773709.993179:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773709.993189:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773709.993193:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773709.993196:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:0.0:1790773709.993202:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773709.993214:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534737792/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773709.993222:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534737280:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773709.993245:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773709.993251:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773709.993254:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773709.993260:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2705 00000100:00080000:1.0:1790773709.993264:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773709.993271:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773709.993275:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534737408:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790773709.993280:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773709.993298:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534737920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773709.993311:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773709.993313:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773709.993320:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773709.993323:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773709.993330:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2705 00000100:00100000:0.0:1790773709.993333:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773709/real 1790773709] req@ffff9c4efac4dc00 x1877759534737408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773709.993336:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773709.993348:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773709.993350:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534737408/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773709.993358:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1790773709.993369:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773709.993374:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773709.993378:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534738048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773709.993380:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534737408:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773709.993392:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534737536:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773709.993399:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773709.993404:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773709.993407:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773709.993413:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2705 00000800:00000100:0.0:1790773709.993415:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773709.993422:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773709.993429:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773709/real 1790773709] req@ffff9c4efac4fb80 x1877759534737536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773709.993433:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773709.993444:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773709.993448:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534737536/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773709.993459:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790773709.993463:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534738176/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773709.993476:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773709.993478:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773709.993483:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773709.993488:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773709.993489:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534737536:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773709.993498:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2705 00000100:00080000:1.0:1790773709.993504:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773709.993505:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534737664:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773709.993513:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773709.993520:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773709.993535:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534738304/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773709.993542:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773709.993564:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773709/real 1790773709] req@ffff9c4efac4ea00 x1877759534737664/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773709.993580:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534737664/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773709.993593:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773709.993597:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773709.993601:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534737664:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773709.993608:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534737792:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773709.993628:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773709.993639:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773709/real 1790773709] req@ffff9c4efac4e680 x1877759534737792/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773709.993682:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534737792/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773709.993694:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773709.993698:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773709.993703:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534737792:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773709.993723:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534737920:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773709.993745:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773709.993757:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773709/real 1790773709] req@ffff9c4efac4f480 x1877759534737920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773709.993770:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534737920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773709.993788:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773709.993791:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773709.993794:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534737920:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773709.993799:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534738048:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773709.993819:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773709.993832:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773709/real 1790773709] req@ffff9c4efac4ed80 x1877759534738048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773709.993844:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534738048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773709.993853:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773709.993856:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773709.993860:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534738048:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773709.993866:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534738176:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773709.993895:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773709.993906:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773709/real 1790773709] req@ffff9c4efac4c700 x1877759534738176/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773709.993919:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534738176/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773709.993931:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773709.993935:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773709.993939:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534738176:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773709.993945:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534738304:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773709.993961:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773709.993971:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773709/real 1790773709] req@ffff9c4efac4f800 x1877759534738304/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773709.993984:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534738304/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773709.993994:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773709.993998:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773709.994001:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534738304:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773710.056738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773710.056740:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773710.056774:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773710.056775:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773710.120770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773710.120794:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773710.120845:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773710.248743:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773710.248854:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773710.312784:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773710.376773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773710.440758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773710.440775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773710.440775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773710.504761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773710.568760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773710.568760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773710.568760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773710.696781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773710.696781:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773710.696803:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773710.696807:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773710.760754:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773710.760755:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773710.760755:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773710.760755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773710.760808:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773710.760825:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773710.888751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773710.952712:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773711.016757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773711.080751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773711.080753:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773711.080757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773711.080787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773711.144770:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773711.144778:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773711.144838:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773711.272771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773711.272771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773711.336727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773711.400749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773711.464750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773711.464750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773711.464767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773711.528770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773711.592771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773711.592780:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773711.592791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773711.720762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773711.720762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773711.720762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773711.720762:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773711.784726:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773711.784734:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773711.784734:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773711.784765:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773711.784831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773711.784861:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773711.912786:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773711.976788:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773712.040774:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773712.104734:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773712.104736:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773712.104755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773712.104789:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773712.168773:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773712.168811:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773712.168819:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773712.296748:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773712.296748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773712.360744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773712.424747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773712.488738:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773712.488738:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773712.488749:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773712.552718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773712.616759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773712.616759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773712.616759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773712.744759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773712.744759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773712.744785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773712.744786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773712.808737:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773712.808737:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773712.808747:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773712.808870:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773712.808877:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773712.808925:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773712.936767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773713.000745:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773713.064785:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773713.128769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773713.128769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773713.128769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773713.128785:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773713.192745:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773713.192747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773713.192899:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773713.320718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773713.320718:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773713.384774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773713.448751:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773713.512759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773713.512759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773713.512759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773713.576757:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773713.640737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773713.640740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773713.640749:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773713.768759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773713.768759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773713.768759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773713.768761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773713.832753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773713.832755:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773713.832758:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773713.832807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773713.832874:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773713.832889:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773713.960813:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773714.024758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773714.088740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773714.152765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773714.152767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773714.152772:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773714.152774:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773714.216763:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773714.216764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773714.216908:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773714.344722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773714.344722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773714.408756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773714.472740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773714.536759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773714.536759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773714.536794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773714.600748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773714.664755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773714.664755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773714.664755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773714.792754:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773714.792754:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773714.792754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773714.792755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773714.856726:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773714.856727:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773714.856740:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773714.856757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773714.856773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773714.856778:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773714.984769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773715.048765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773715.112730:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773715.112739:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1790773715.112742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773715.112743:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773715.112754:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2711 00000100:00080000:1.0:1790773715.112763:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773715.112773:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773715.112806:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534738432/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773715.112832:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773715.112836:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773715.112839:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773715.112843:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2711 00000100:00080000:1.0:1790773715.112847:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773715.112853:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773715.112863:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773715.112878:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534738560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773715.112889:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773715.112891:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534738432:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773715.112893:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773715.112896:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773715.112900:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2711 00000100:00080000:1.0:1790773715.112905:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773715.112919:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773715.112924:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773715.112935:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534738688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773715.112944:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773715.112948:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773715.112950:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773715.112954:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2711 00000800:00000100:0.0:1790773715.112955:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773715.112958:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773715.112963:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773715.112969:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773715.112980:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534738816/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773715.112991:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773715.112995:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773715.112997:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773715.112999:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773715/real 1790773715] req@ffff9c4efac4df80 x1877759534738432/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773715.113001:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2711 00000100:00080000:1.0:1790773715.113005:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773715.113010:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773715.113014:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773715.113018:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534738432/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773715.113024:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534738944/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773715.113029:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773715.113034:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773715.113035:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773715.113039:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1790773715.113040:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534738432:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773715.113041:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773715.113045:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2711 00000100:00080000:1.0:1790773715.113048:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773715.113053:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773715.113058:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773715.113062:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534738560:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773715.113067:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534739072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773715.113077:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773715.113081:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773715.113083:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773715.113088:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2711 00000100:00080000:1.0:1790773715.113091:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773715.113092:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773715.113096:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773715.113101:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773715.113105:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773715/real 1790773715] req@ffff9c4efac4f800 x1877759534738560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773715.113112:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534739200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773715.113120:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534738560/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773715.113122:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773715.113126:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773715.113128:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773715.113150:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2711 00000100:00080000:0.0:1790773715.113153:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773715.113154:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773715.113157:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773715.113160:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773715.113161:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534738560:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790773715.113165:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773715.113168:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534738688:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773715.113177:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534739328/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773715.113187:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773715.113188:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773715.113191:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773715.113194:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773715.113198:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2711 00000100:00100000:0.0:1790773715.113199:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773715/real 1790773715] req@ffff9c4efac4c700 x1877759534738688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773715.113202:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773715.113207:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773715.113212:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534738688/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773715.113212:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790773715.113222:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790773715.113222:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534739456/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773715.113226:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773715.113230:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534738688:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773715.113236:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534738816:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773715.113258:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773715.113270:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773715/real 1790773715] req@ffff9c4efac4ed80 x1877759534738816/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773715.113280:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534738816/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773715.113287:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773715.113290:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773715.113292:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534738816:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773715.113296:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534738944:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773715.113319:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773715.113326:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773715/real 1790773715] req@ffff9c4efac4f480 x1877759534738944/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773715.113335:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534738944/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773715.113345:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773715.113349:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773715.113352:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534738944:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773715.113379:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534739072:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773715.113402:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773715.113411:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773715/real 1790773715] req@ffff9c4efac4e680 x1877759534739072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773715.113419:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534739072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773715.113436:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773715.113438:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773715.113441:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534739072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773715.113445:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534739200:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773715.113456:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773715.113462:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773715/real 1790773715] req@ffff9c4efac4ea00 x1877759534739200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773715.113478:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534739200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773715.113485:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773715.113487:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773715.113489:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534739200:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773715.113493:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534739328:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773715.113504:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773715.113511:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773715/real 1790773715] req@ffff9c4efac4fb80 x1877759534739328/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773715.113519:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534739328/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773715.113526:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773715.113528:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773715.113530:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534739328:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773715.113534:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534739456:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773715.113544:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773715.113550:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773715/real 1790773715] req@ffff9c4efac4dc00 x1877759534739456/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773715.113557:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534739456/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773715.113564:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773715.113565:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773715.113568:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534739456:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773715.176760:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773715.176761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773715.176761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773715.176761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773715.240761:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773715.240768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773715.240821:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773715.368763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773715.368763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773715.432757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773715.496741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773715.560761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773715.560761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773715.560768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773715.624751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773715.688728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773715.688729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773715.688737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773715.816755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773715.816756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773715.816757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773715.816757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773715.880746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773715.880746:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773715.880749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773715.880752:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773715.880782:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773715.880784:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773716.008724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773716.072779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773716.136706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773716.200764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773716.200764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773716.200764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773716.200765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773716.264739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773716.264747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773716.264802:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773716.392723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773716.392725:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773716.456765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773716.520754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773716.584768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773716.584777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773716.584869:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773716.648763:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773716.712750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773716.712750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773716.712764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773716.840737:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773716.840737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773716.840737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773716.840768:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773716.904736:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773716.904736:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773716.904784:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773716.904785:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773716.904806:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773716.904814:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773717.032735:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773717.096741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773717.160744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773717.224768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773717.224769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773717.224777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773717.224862:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773717.288735:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773717.288767:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773717.288780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773717.416709:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773717.416725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773717.480776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773717.544734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773717.608740:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773717.608741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773717.608742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773717.672741:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773717.736725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773717.736728:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773717.736737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773717.864737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773717.864741:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773717.864771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773717.864773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773717.928754:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773717.928754:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773717.928754:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773717.928802:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773717.928809:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773717.928816:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773718.056756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773718.120752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773718.184776:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773718.248767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773718.248768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773718.248769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773718.248780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773718.312772:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773718.312774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773718.312800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773718.440737:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773718.440757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773718.504738:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773718.568862:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773718.632760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773718.632760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773718.632799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773718.696781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773718.760733:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773718.760733:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773718.760747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773718.888726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773718.888731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773718.888732:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773718.888758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773718.952800:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773718.952800:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773718.952800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773718.952806:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773718.952847:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773718.952848:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773719.080727:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773719.144755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773719.208747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773719.272736:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773719.272738:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773719.272739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773719.272741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773719.336755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773719.336755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773719.336820:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773719.464775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773719.464782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773719.528756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773719.592738:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773719.656744:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773719.656744:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773719.656775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773719.720782:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773719.784757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773719.784757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773719.784757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773719.912734:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773719.912735:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773719.912794:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773719.912802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773719.976779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773719.976779:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773719.976779:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773719.976796:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773719.976839:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773719.976843:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773720.104822:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773720.168731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773720.232767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773720.232783:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773720.232795:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773720.232802:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773720.232815:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2716 00000100:00080000:1.0:1790773720.232830:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773720.232845:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773720.232897:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534739584/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773720.232930:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773720.232934:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773720.232936:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773720.232939:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2716 00000100:00080000:1.0:1790773720.232943:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773720.232947:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773720.232958:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773720.232971:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534739712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773720.232980:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773720.232984:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1790773720.232987:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534739584:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773720.232987:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773720.232995:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2716 00000100:00080000:1.0:1790773720.232998:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773720.233001:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773720.233004:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773720.233015:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534739840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773720.233022:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773720.233026:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773720.233027:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773720.233046:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2716 00000100:00080000:1.0:1790773720.233063:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773720.233069:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773720.233076:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773720.233090:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534739968/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773720.233102:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773720.233107:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773720.233110:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773720.233116:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2716 00000100:00080000:1.0:1790773720.233122:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773720.233127:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534739712:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773720.233130:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773720.233138:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773720.233151:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534740096/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773720.233161:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773720.233164:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534739840:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773720.233166:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773720.233170:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773720.233175:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2716 00000100:00080000:1.0:1790773720.233191:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773720.233195:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534739968:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773720.233197:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773720.233204:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773720.233215:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534740224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773720.233222:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534740096:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773720.233225:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773720.233230:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773720.233232:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773720.233237:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2716 00000100:00080000:1.0:1790773720.233241:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773720.233245:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773720.233248:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773720.233249:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534740224:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773720.233260:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534740352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773720.233275:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773720.233278:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773720.233280:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773720.233284:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2716 00000100:00080000:1.0:1790773720.233288:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773720.233290:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534740352:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773720.233295:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773720.233301:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773720.233310:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534740480/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773720.233318:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773720.233322:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773720.233324:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773720.233327:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534740480:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773720.233330:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2716 00000100:00080000:1.0:1790773720.233336:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773720.233341:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773720.233345:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773720.233357:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534740608/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773720.233383:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534740608:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773720.296740:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773720.296778:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773720.296779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773720.296779:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773720.360778:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773720.360841:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773720.360859:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773720.488722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773720.488723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773720.552743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773720.616746:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773720.680728:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773720.680728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773720.680728:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773720.744854:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773720.808744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773720.808744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773720.808764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773720.936712:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773720.936733:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773720.936738:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773720.936787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773721.000724:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773721.000757:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773721.000787:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773721.000787:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773721.000787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773721.000811:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773721.128771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773721.192788:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773721.256742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773721.320769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773721.320769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773721.320769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773721.320784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773721.384740:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773721.384744:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773721.384819:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773721.512763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773721.512763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773721.576740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773721.640737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773721.704717:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773721.704718:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773721.704732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773721.768739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773721.832757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773721.832757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773721.832758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773721.960745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773721.960745:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773721.960746:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773721.960748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773722.024727:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773722.024728:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773722.024732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773722.024732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773722.024766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773722.024767:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773722.152775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773722.216733:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773722.280744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773722.344734:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773722.344737:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773722.344751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773722.344783:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773722.408735:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773722.408758:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773722.408881:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773722.536760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773722.536760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773722.600762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773722.664758:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773722.728770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773722.728771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773722.728771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773722.792759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773722.856721:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773722.856733:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773722.856807:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773722.984758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773722.984758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773722.984758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773722.984761:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773723.048759:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773723.048759:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773723.048759:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773723.048760:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773723.048793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773723.048799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773723.176755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773723.240736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773723.304739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:3.0:1790773723.304874:0:2386:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:3.0:1790773723.304912:0:2386:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1790773723.304929:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773723.305001:0:2386:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:3.0:1790773723.305016:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773723.305027:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773723.305037:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773723.305047:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773723.305060:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773723.305073:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773723.305076:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773720/real 1790773723] req@ffff9c4efac4f800 x1877759534740608/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1790773723.305086:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:0.0:1790773723.305113:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534740608/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773723.305142:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773723.305149:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773723.305154:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534740608:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773723.305162:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773720/real 1790773723] req@ffff9c4efac4c700 x1877759534740480/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773723.305193:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534740480/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773723.305206:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773723.305211:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773723.305215:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534740480:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773723.305224:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773720/real 1790773723] req@ffff9c4efac4ed80 x1877759534740352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773723.305238:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534740352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773723.305247:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773723.305251:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773723.305254:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534740352:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773723.305263:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773720/real 1790773723] req@ffff9c4efac4f480 x1877759534740224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773723.305274:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534740224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773723.305292:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773723.305296:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773723.305300:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534740224:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773723.305308:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773720/real 1790773723] req@ffff9c4efac4e680 x1877759534740096/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773723.305322:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534740096/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773723.305332:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773723.305336:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773723.305341:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534740096:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773723.305348:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773720/real 1790773723] req@ffff9c4efac4dc00 x1877759534739712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773723.305361:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534739712/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773723.305376:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773723.305383:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773723.305386:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534739712:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773723.305393:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773720/real 1790773723] req@ffff9c4efac4fb80 x1877759534739840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773723.305404:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534739840/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773723.305414:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773723.305418:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773723.305422:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534739840:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773723.305429:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773720/real 1790773723] req@ffff9c4efac4ea00 x1877759534739968/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773723.305441:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534739968/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773723.305450:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773723.305454:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773723.305457:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534739968:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773723.305464:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773720/real 1790773723] req@ffff9c4efac4e300 x1877759534739584/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773723.305477:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534739584/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773723.305485:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773723.305488:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773723.305492:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534739584:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773723.368752:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773723.368753:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773723.368753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773723.368758:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773723.432742:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773723.432742:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773723.432790:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773723.560770:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773723.560785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773723.624785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773723.688734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773723.752816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773723.752817:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773723.752817:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773723.816742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773723.880751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773723.880751:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773723.880751:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773724.008750:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773724.008751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773724.008751:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773724.008764:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773724.072760:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773724.072760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773724.072794:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773724.072817:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773724.072880:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773724.072911:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773724.200778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773724.264778:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773724.328768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773724.392732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773724.392732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773724.392737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773724.392771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773724.456761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773724.456765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773724.456851:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773724.584756:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773724.584759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773724.648764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773724.712745:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773724.776763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773724.776763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773724.776763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773724.840771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773724.904811:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773724.904812:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773724.904813:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773725.032719:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773725.032725:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773725.032753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773725.032760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773725.096765:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773725.096771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773725.096794:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773725.096799:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773725.096834:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773725.096841:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773725.224723:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773725.288736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773725.352814:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773725.352823:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773725.352826:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773725.352837:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773725.352851:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2721 00000100:00080000:1.0:1790773725.352863:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773725.352877:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773725.352919:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534740736/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773725.352965:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773725.352969:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773725.352972:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773725.352976:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2721 00000100:00080000:1.0:1790773725.352981:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773725.352986:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773725.352996:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773725.353013:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534740864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773725.353020:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534740736:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773725.353030:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773725.353036:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773725.353038:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773725.353044:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2721 00000100:00080000:1.0:1790773725.353049:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773725.353054:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773725.353061:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773725.353075:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534740992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773725.353102:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773725.353106:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773725.353112:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773725.353115:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773725.353121:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2721 00000100:00080000:1.0:1790773725.353134:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773725.353141:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773725.353149:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773725.353166:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534741120/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773725.353178:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773725.353181:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773725/real 1790773725] req@ffff9c4efac4df80 x1877759534740736/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773725.353183:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773725.353188:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773725.353201:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2721 00000100:00080000:1.0:1790773725.353208:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773725.353217:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534740736/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773725.353217:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773725.353227:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:0.0:1790773725.353238:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1790773725.353243:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534741248/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773725.353247:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773725.353257:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534740736:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773725.353265:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773725.353271:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773725.353273:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773725.353279:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2721 00000100:00080000:1.0:1790773725.353283:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773725.353290:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773725.353294:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534740864:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790773725.353298:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773725.353315:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534741376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773725.353327:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1790773725.353332:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773725.353334:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773725.353339:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773725.353345:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2721 00000100:00080000:1.0:1790773725.353350:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773725.353356:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773725.353365:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773725/real 1790773725] req@ffff9c4efac4e300 x1877759534740864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773725.353368:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773725.353385:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534741504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773725.353391:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534740864/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773725.353408:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773725.353414:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773725.353416:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773725.353423:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773725.353431:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773725.353435:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534740864:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773725.353437:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2721 00000100:00080000:1.0:1790773725.353447:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773725.353450:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534740992:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773725.353456:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773725.353465:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773725.353481:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534741632/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773725.353484:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773725.353497:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773725.353501:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1790773725.353505:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773725/real 1790773725] req@ffff9c4efac4ea00 x1877759534740992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773725.353506:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773725.353515:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2721 00000100:00080000:1.0:1790773725.353525:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773725.353530:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534740992/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773725.353534:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773725.353544:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:0.0:1790773725.353547:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773725.353554:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773725.353563:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534740992:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773725.353564:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534741760/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773725.353584:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534741120:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773725.353611:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773725.353623:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773725/real 1790773725] req@ffff9c4efac4fb80 x1877759534741120/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773725.353635:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534741120/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773725.353645:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773725.353648:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773725.353683:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534741120:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773725.353689:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534741248:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773725.353725:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773725.353736:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773725/real 1790773725] req@ffff9c4efac4dc00 x1877759534741248/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773725.353747:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534741248/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773725.353757:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773725.353761:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773725.353768:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534741248:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773725.353792:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534741376:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773725.353816:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773725.353837:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773725/real 1790773725] req@ffff9c4efac4e680 x1877759534741376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773725.353851:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534741376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773725.353868:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773725.353872:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773725.353875:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534741376:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773725.353882:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534741504:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773725.353911:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773725.353924:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773725/real 1790773725] req@ffff9c4efac4f480 x1877759534741504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773725.353960:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534741504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773725.353970:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773725.353974:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773725.353978:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534741504:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773725.353984:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534741632:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773725.354007:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773725.354020:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773725/real 1790773725] req@ffff9c4efac4ed80 x1877759534741632/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773725.354033:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534741632/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773725.354046:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773725.354050:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773725.354054:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534741632:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773725.354060:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534741760:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773725.354082:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773725.354094:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773725/real 1790773725] req@ffff9c4efac4c700 x1877759534741760/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773725.354107:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534741760/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773725.354124:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773725.354128:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773725.354131:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534741760:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773725.416776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773725.416791:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773725.416792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773725.416792:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773725.480764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773725.480814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773725.481022:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773725.608750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773725.608750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773725.672781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773725.736822:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773725.800809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773725.800812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773725.800812:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773725.864758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773725.928764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773725.928764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773725.928764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773726.056708:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773726.056715:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773726.056736:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773726.056786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773726.120736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773726.120737:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773726.120771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773726.120806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773726.120821:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773726.120853:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773726.248741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773726.312725:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773726.376792:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773726.440725:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773726.440726:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773726.440734:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773726.440776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773726.504722:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773726.504752:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773726.504836:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773726.632782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773726.632782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773726.699102:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773726.760759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773726.824806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773726.824808:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773726.824813:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773726.888714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773726.952760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773726.952760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773726.952760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773727.080773:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773727.080774:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773727.080784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773727.080784:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773727.144745:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773727.144766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773727.144850:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773727.144861:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773727.144926:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773727.144951:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773727.272745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773727.336765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773727.400719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773727.464759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773727.464759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773727.464760:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773727.464781:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773727.528746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773727.528765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773727.528796:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773727.656713:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773727.656721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773727.720726:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773727.784750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773727.848751:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773727.848751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773727.848752:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773727.912729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773727.976772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773727.976772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773727.976772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773728.104767:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773728.104767:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773728.104767:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773728.104767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773728.168743:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773728.168743:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773728.168749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773728.168751:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773728.168782:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773728.168800:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773728.296717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773728.360739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773728.424743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773728.488763:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773728.488763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773728.488764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773728.488764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773728.552769:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773728.552781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773728.552783:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773728.680712:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773728.680718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773728.744769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773728.808734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773728.872764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773728.872764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773728.872880:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773728.936753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773729.000731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773729.000732:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773729.000732:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773729.128757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773729.128758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773729.128758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773729.128758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773729.192764:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773729.192764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773729.192786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773729.192806:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773729.192829:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773729.192836:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773729.320739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773729.384755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773729.448770:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773729.512764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773729.512764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773729.512765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773729.512765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773729.576761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773729.576779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773729.576816:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773729.704726:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773729.704726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773729.768774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773729.832743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773729.896760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773729.896760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773729.896761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773729.960738:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773730.024756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773730.024756:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773730.024865:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773730.152737:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773730.152737:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773730.152737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773730.152737:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773730.216749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773730.216750:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773730.216753:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773730.216760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773730.216784:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773730.216789:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773730.344741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773730.408714:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773730.472760:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773730.472762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773730.472773:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773730.472779:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773730.472790:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2726 00000100:00080000:1.0:1790773730.472801:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773730.472814:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773730.472871:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534741888/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773730.472909:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773730.472913:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773730.472915:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773730.472919:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2726 00000100:00080000:1.0:1790773730.472924:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773730.472930:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773730.472940:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773730.472960:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534741888:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773730.472961:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534742016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773730.472978:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773730.472982:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773730.472985:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773730.472990:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2726 00000100:00080000:1.0:1790773730.472995:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773730.473016:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:0.0:1790773730.473023:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773730.473023:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773730.473042:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534742144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773730.473054:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773730.473058:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773730.473060:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773730.473064:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773730/real 1790773730] req@ffff9c4efac4f800 x1877759534741888/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773730.473069:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2726 00000100:00080000:1.0:1790773730.473079:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773730.473088:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534741888/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773730.473097:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:0.0:1790773730.473098:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773730.473107:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790773730.473109:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773730.473111:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534741888:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773730.473131:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534742272/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773730.473137:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534742016:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773730.473160:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773730.473169:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773730.473173:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773730/real 1790773730] req@ffff9c4efac4c700 x1877759534742016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773730.473176:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773730.473182:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773730.473192:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2726 00000100:00100000:0.0:1790773730.473193:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534742016/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773730.473200:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773730.473212:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773730.473219:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773730.473234:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534742400/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773730.473253:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773730.473259:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773730.473260:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773730.473262:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773730.473267:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773730.473272:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2726 00000100:00100000:0.0:1790773730.473275:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534742016:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773730.473280:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773730.473284:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534742144:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773730.473291:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773730.473299:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1790773730.473304:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773730.473314:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773730/real 1790773730] req@ffff9c4efac4ed80 x1877759534742144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773730.473319:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534742528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773730.473330:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534742144/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773730.473361:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773730.473365:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773730.473368:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534742144:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773730.473372:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773730.473378:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773730.473380:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1790773730.473381:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534742272:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773730.473392:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2726 00000100:00080000:1.0:1790773730.473400:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1790773730.473402:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773730.473411:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773730.473417:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773730/real 1790773730] req@ffff9c4efac4f480 x1877759534742272/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1790773730.473420:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773730.473444:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534742656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773730.473445:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534742272/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773730.473479:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773730.473487:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773730.473490:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773730.473492:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773730.473497:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2726 00000100:00080000:0.0:1790773730.473498:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773730.473505:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773730.473506:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534742272:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773730.473518:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534742400:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773730.473520:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773730.473529:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:0.0:1790773730.473542:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773730.473547:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534742784/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773730.473556:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773730/real 1790773730] req@ffff9c4efac4e680 x1877759534742400/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773730.473563:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1790773730.473569:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534742400/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773730.473569:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773730.473574:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773730.473585:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773730.473586:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2726 00000100:00080000:0.0:1790773730.473592:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773730.473596:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534742400:192.168.204.150@tcp:8:kworker.0 00000100:00080000:1.0:1790773730.473596:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1790773730.473609:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534742528:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773730.473609:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773730.473622:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773730.473641:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534742912/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1790773730.473683:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773730.473703:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773730/real 1790773730] req@ffff9c4efac4dc00 x1877759534742528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773730.473715:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534742528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773730.473731:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773730.473735:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773730.473738:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534742528:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773730.473754:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534742656:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773730.473774:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773730.473785:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773730/real 1790773730] req@ffff9c4efac4fb80 x1877759534742656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773730.473797:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534742656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773730.473805:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773730.473808:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773730.473811:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534742656:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773730.473817:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534742784:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773730.473847:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773730.473860:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773730/real 1790773730] req@ffff9c4efac4ea00 x1877759534742784/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773730.473871:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534742784/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773730.473882:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773730.473886:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773730.473890:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534742784:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773730.473897:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534742912:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773730.473914:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773730.473923:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773730/real 1790773730] req@ffff9c4efac4e300 x1877759534742912/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773730.473931:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534742912/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773730.473938:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773730.473940:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773730.473943:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534742912:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773730.536762:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773730.536763:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773730.536763:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773730.536763:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773730.600750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773730.600767:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773730.600800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773730.728760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773730.728760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773730.792740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773730.856732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773730.920771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773730.920771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773730.920771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773730.984756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773731.048740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773731.048741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773731.048741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773731.176758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773731.176758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773731.176759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773731.176759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773731.240697:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773731.240719:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773731.240736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773731.240742:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773731.240747:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773731.240774:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773731.368750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773731.432792:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773731.496746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773731.560748:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773731.560749:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773731.560751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773731.560752:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773731.624778:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773731.624835:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773731.624985:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773731.752769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773731.752769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773731.816764:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773731.880760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773731.944750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773731.944750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773731.944750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773732.008722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773732.072761:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773732.072761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773732.072761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773732.200782:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773732.200782:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773732.200782:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773732.200782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773732.264739:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773732.264739:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773732.264744:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773732.264745:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773732.264773:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773732.264776:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773732.392724:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773732.456724:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773732.520821:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773732.584767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773732.584767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773732.584769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773732.584769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773732.648755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773732.648782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773732.648820:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773732.776738:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773732.776742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773732.840749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773732.904740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773732.968776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773732.968776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773732.968776:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773733.032740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773733.096762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773733.096762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773733.096763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773733.224763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773733.224764:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773733.224764:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773733.224771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773733.288790:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773733.288791:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773733.288794:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773733.288795:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773733.288829:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773733.288831:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773733.416748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773733.480790:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773733.544733:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773733.608744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773733.608746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773733.608746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773733.608752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773733.672756:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773733.672756:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773733.672825:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773733.800750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773733.800750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773733.864728:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773733.928730:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773733.992741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773733.992741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773733.992747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773734.056754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773734.120747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773734.120747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773734.120747:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773734.248738:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773734.248738:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773734.248739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773734.248744:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773734.312740:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773734.312740:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773734.312744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773734.312748:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773734.312793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773734.312810:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773734.440719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773734.504767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773734.568728:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773734.632764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773734.632764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773734.632765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773734.632790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773734.696746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773734.696797:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773734.696831:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773734.824742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773734.824743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773734.888795:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773734.952739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773735.016771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773735.016771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773735.016777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773735.080764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773735.144736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773735.144737:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773735.144737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773735.272769:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773735.272771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773735.272772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773735.272773:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773735.336725:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773735.336727:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773735.336735:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773735.336769:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773735.336787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773735.336792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773735.464763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773735.528784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773735.592756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773735.592769:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773735.592776:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773735.592781:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773735.592794:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2731 00000100:00080000:0.0:1790773735.592805:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773735.592816:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773735.592853:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534743040/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773735.592882:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773735.592886:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773735.592889:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773735.592893:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2731 00000100:00080000:0.0:1790773735.592898:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773735.592905:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773735.592914:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773735.592926:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534743040:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773735.592939:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534743168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773735.592953:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773735.592957:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773735.592961:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773735.592966:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2731 00000100:00080000:0.0:1790773735.592994:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773735.593000:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773735.593007:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773735.593008:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773735.593021:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534743296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773735.593030:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773735.593033:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773735.593034:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773735.593044:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2731 00000100:00080000:0.0:1790773735.593047:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773735.593052:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773735.593056:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773735.593058:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773735/real 1790773735] req@ffff9c4ec7a7f480 x1877759534743040/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773735.593067:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534743424/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773735.593085:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773735.593088:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773735.593090:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773735.593095:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2731 00000100:00100000:1.0:1790773735.593097:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534743040/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773735.593098:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773735.593103:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773735.593108:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773735.593119:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534743552/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773735.593121:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773735.593133:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773735.593135:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773735.593142:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1790773735.593144:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534743040:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773735.593146:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773735.593156:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2731 00000100:00080000:0.0:1790773735.593161:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773735.593167:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773735.593173:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773735.593176:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534743168:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773735.593189:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534743680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773735.593199:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773735.593203:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773735.593204:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773735.593210:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773735.593211:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2731 00000100:00080000:0.0:1790773735.593219:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773735.593226:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773735.593232:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773735/real 1790773735] req@ffff9c4ec7a7df80 x1877759534743168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773735.593233:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773735.593252:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534743808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773735.593257:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534743168/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773735.593275:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773735.593281:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773735.593284:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773735.593289:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773735.593290:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2731 00000100:00080000:0.0:1790773735.593299:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773735.593299:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773735.593310:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773735.593311:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534743168:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773735.593319:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773735.593324:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534743296:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773735.593337:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534743936/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773735.593349:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773735.593355:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773735.593357:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773735.593361:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773735.593368:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2731 00000100:00080000:0.0:1790773735.593373:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773735.593374:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773735/real 1790773735] req@ffff9c4ec7a7ed80 x1877759534743296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773735.593383:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773735.593394:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773735.593400:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534743296/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773735.593408:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534744064/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773735.593419:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773735.593427:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773735.593433:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534743296:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773735.593442:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534743424:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773735.593465:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773735.593478:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773735/real 1790773735] req@ffff9c4ec7a7d880 x1877759534743424/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773735.593492:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534743424/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773735.593504:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773735.593508:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773735.593512:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534743424:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773735.593519:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534743552:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773735.593539:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773735.593551:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773735/real 1790773735] req@ffff9c4ec7a7c380 x1877759534743552/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773735.593565:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534743552/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773735.593575:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773735.593579:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773735.593583:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534743552:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773735.593601:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534743680:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773735.593622:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773735.593634:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773735/real 1790773735] req@ffff9c4ec7a7ca80 x1877759534743680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773735.593648:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534743680/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773735.593686:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773735.593690:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773735.593694:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534743680:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773735.593701:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534743808:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773735.593723:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773735.593749:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773735/real 1790773735] req@ffff9c4ec7a7fb80 x1877759534743808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773735.593763:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534743808/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773735.593774:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773735.593778:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773735.593781:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534743808:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773735.593788:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534743936:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773735.593809:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773735.593821:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773735/real 1790773735] req@ffff9c4ec7a7e300 x1877759534743936/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773735.593834:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534743936/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773735.593846:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773735.593850:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773735.593854:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534743936:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773735.593861:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534744064:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773735.593881:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773735.593893:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773735/real 1790773735] req@ffff9c4ec7a7c700 x1877759534744064/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773735.593906:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534744064/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773735.593917:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773735.593921:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773735.593925:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534744064:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773735.656761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773735.656761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773735.656761:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773735.656820:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773735.720760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773735.720804:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773735.720804:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773735.848750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773735.848750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773735.912777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773735.976769:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773736.040745:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773736.040762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773736.040764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773736.104758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773736.168774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773736.168774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773736.168775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773736.296767:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773736.296767:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773736.296767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773736.296768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773736.360731:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773736.360745:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773736.360759:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773736.360764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773736.360780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773736.360789:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773736.488773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773736.552768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773736.616756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773736.680759:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773736.680759:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773736.680765:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773736.680861:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773736.744765:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773736.744788:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773736.744819:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773736.872754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773736.872760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773736.936772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773737.000775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773737.064771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773737.064772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773737.064772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773737.128801:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773737.192745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773737.192745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773737.192745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773737.320764:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773737.320764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773737.320764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773737.320865:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773737.384775:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773737.384775:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773737.384777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773737.384777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773737.384806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773737.384815:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773737.512754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773737.576773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773737.640798:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773737.704769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773737.704770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773737.704770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773737.704771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773737.768792:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773737.768793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773737.768864:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773737.896785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773737.896785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773737.960741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773738.024745:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773738.088790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773738.088790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773738.088790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773738.152767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773738.216787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773738.216787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773738.216787:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773738.344729:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773738.344735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773738.344742:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773738.344752:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773738.408782:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773738.408792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773738.408808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773738.408851:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773738.408851:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773738.408854:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773738.536807:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773738.600738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773738.664769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773738.728799:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773738.728800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773738.728810:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773738.728862:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773738.792742:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773738.792783:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773738.792859:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773738.920787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773738.920787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773738.984771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773739.048787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773739.112759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773739.112760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773739.112787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773739.176752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773739.240768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773739.240768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773739.240768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773739.368759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773739.368759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773739.368759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773739.368787:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773739.432756:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773739.432757:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773739.432757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773739.432789:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773739.432814:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773739.432818:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773739.560720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773739.624776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773739.688736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773739.752765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773739.752765:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773739.752768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773739.752803:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773739.816779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773739.816817:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773739.816839:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773739.944743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773739.944775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773740.008788:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773740.072773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773740.136789:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773740.136790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773740.136790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773740.200748:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773740.264806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773740.264807:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773740.264807:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773740.392783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773740.392783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773740.393037:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773740.393724:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773740.456800:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773740.456800:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773740.456800:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773740.456963:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773740.456981:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773740.457015:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773740.584783:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773740.648746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773740.712853:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773740.712864:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773740.712868:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773740.712878:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2736 00000100:00080000:0.0:1790773740.712888:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773740.712899:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773740.712945:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534744192/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773740.712971:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773740.712975:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773740.712977:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773740.712981:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2736 00000100:00080000:0.0:1790773740.712985:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773740.712991:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773740.713001:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773740.713017:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534744320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773740.713030:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773740.713034:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773740.713039:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773740.713042:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534744192:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773740.713051:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2736 00000100:00080000:0.0:1790773740.713057:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773740.713063:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773740.713070:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773740.713087:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534744448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773740.713098:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773740.713101:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773740.713104:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773740.713109:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2736 00000100:00080000:0.0:1790773740.713115:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773740.713120:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773740.713126:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773740.713129:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773740.713171:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773740/real 1790773740] req@ffff9c4ec7a7d180 x1877759534744192/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773740.713187:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534744192/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773740.713197:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773740.713200:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773740.713204:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534744192:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773740.713215:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534744576/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773740.713227:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534744320:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773740.713232:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773740.713238:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773740.713240:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773740.713246:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773740.713251:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2736 00000100:00080000:0.0:1790773740.713259:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773740.713262:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773740/real 1790773740] req@ffff9c4ec7a7c700 x1877759534744320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773740.713273:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773740.713281:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773740.713286:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534744320/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773740.713303:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773740.713311:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534744704/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773740.713311:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773740.713321:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534744320:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773740.713347:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773740.713349:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534744448:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773740.713355:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773740.713360:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773740.713368:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2736 00000100:00080000:0.0:1790773740.713381:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773740.713384:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773740.713392:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773740.713400:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773740.713405:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773740/real 1790773740] req@ffff9c4ec7a7e300 x1877759534744448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773740.713419:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534744832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773740.713447:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534744448/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773740.713451:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773740.713465:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773740.713468:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773740.713471:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773740.713477:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773740.713484:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2736 00000100:00100000:1.0:1790773740.713489:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534744448:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773740.713492:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773740.713501:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773740.713507:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773740.713513:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534744576:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773740.713522:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534744960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773740.713536:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773740.713540:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773740.713543:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773740.713551:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773740.713552:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2736 00000100:00080000:0.0:1790773740.713561:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773740.713569:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773740/real 1790773740] req@ffff9c4ec7a7fb80 x1877759534744576/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773740.713571:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773740.713588:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773740.713596:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534744576/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773740.713606:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534745088/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773740.713611:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773740.713623:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773740.713626:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773740.713646:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534744576:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773740.713686:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773740.713694:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773740.713701:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534744704:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773740.713705:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2736 00000100:00080000:0.0:1790773740.713712:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773740.713720:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773740.713728:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773740.713728:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000800:00000100:1.0:1790773740.713740:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773740.713757:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534745216/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773740.713772:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773740/real 1790773740] req@ffff9c4ec7a7ca80 x1877759534744704/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773740.713789:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534744704/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773740.713811:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773740.713816:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773740.713820:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534744704:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773740.713826:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534744832:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773740.713849:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773740.713871:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773740/real 1790773740] req@ffff9c4ec7a7c380 x1877759534744832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773740.713881:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534744832/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773740.713897:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773740.713901:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773740.713905:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534744832:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773740.713923:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534744960:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773740.713941:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773740.713949:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773740/real 1790773740] req@ffff9c4ec7a7d880 x1877759534744960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773740.713957:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534744960/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773740.713962:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773740.713965:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773740.713968:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534744960:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773740.713973:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534745088:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773740.713990:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773740.714001:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773740/real 1790773740] req@ffff9c4ec7a7ed80 x1877759534745088/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773740.714013:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534745088/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773740.714023:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773740.714026:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773740.714029:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534745088:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773740.714033:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534745216:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773740.714047:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773740.714055:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773740/real 1790773740] req@ffff9c4ec7a7df80 x1877759534745216/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773740.714063:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534745216/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773740.714069:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773740.714071:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773740.714073:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534745216:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773740.776764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773740.776764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773740.776764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773740.776779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773740.840760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773740.840776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773740.840892:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773740.968728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773740.968728:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773741.032777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773741.096759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773741.160720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773741.160722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773741.160753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773741.224730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773741.288753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773741.288753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773741.288754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773741.416759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773741.416760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773741.416760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773741.416763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773741.480754:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773741.480761:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773741.480761:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773741.480899:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773741.480908:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773741.480973:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773741.608710:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773741.672762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773741.736734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773741.800747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773741.800748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773741.800748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773741.800748:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773741.864743:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773741.864763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773741.864795:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773741.992716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773741.992731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773742.056756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773742.120744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773742.184753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773742.184753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773742.184753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773742.248727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773742.312724:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773742.312724:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773742.312726:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773742.440749:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773742.440750:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773742.440750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773742.440852:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773742.504719:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773742.504720:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773742.504721:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773742.504725:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773742.504749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773742.504769:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773742.632745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773742.696752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773742.760724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773742.824733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773742.824734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773742.824734:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773742.824771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773742.888737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773742.888749:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773742.888772:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773743.016746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773743.016746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773743.080741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773743.144729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773743.208753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773743.208753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773743.208753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773743.272743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773743.336746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773743.336746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773743.336747:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773743.464737:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773743.464738:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773743.464738:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773743.464837:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773743.528750:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773743.528751:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773743.528751:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773743.528758:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773743.528778:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773743.528786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773743.656763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773743.720790:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773743.784774:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773743.848740:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773743.848740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773743.848740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773743.848742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773743.912721:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773743.912744:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773743.912797:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773744.040745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773744.040745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773744.104746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773744.168748:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773744.232757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773744.232763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773744.232764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773744.296754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773744.360750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773744.360751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773744.360760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773744.488772:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773744.488772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773744.488784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773744.488784:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773744.552764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773744.552764:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773744.552770:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773744.552795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773744.552805:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773744.552809:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773744.680946:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773744.744762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773744.808764:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773744.872745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773744.872745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773744.872745:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773744.872747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773744.936746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773744.936753:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773744.936792:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773745.064735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773745.064735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773745.128742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773745.192734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773745.256743:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773745.256743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773745.256746:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773745.320727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773745.384746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773745.384746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773745.384748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773745.512721:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773745.512721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773745.512723:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773745.512821:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773745.576739:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773745.576740:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773745.576740:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773745.576741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773745.576773:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773745.576775:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773745.704763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773745.768759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773745.832757:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773745.832762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773745.832769:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773745.832775:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773745.832785:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2741 00000100:00080000:3.0:1790773745.832796:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773745.832807:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773745.832841:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534745344/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773745.832863:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773745.832865:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773745.832867:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773745.832870:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2741 00000100:00080000:3.0:1790773745.832873:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773745.832876:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773745.832883:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773745.832893:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534745472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773745.832900:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773745.832903:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773745.832904:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773745.832907:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2741 00000100:00080000:3.0:1790773745.832909:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773745.832912:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773745.832915:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773745.832920:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534745344:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773745.832924:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534745600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773745.832933:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773745.832935:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773745.832936:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773745.832939:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2741 00000100:00080000:3.0:1790773745.832941:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773745.832944:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773745.832947:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773745.832953:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534745728/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773745.832959:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773745.832961:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773745.832962:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773745.832965:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2741 00000100:00080000:3.0:1790773745.832967:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773745.832970:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773745.832972:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773745.832979:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534745856/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773745.832985:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773745.832990:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773745.832990:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773745.832993:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773745.832997:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2741 00000100:00080000:3.0:1790773745.832999:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773745.833003:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773745.833006:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773745.833012:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534745984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773745.833019:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773745.833021:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773745.833022:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773745.833025:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2741 00000100:00080000:3.0:1790773745.833027:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773745.833030:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773745.833033:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773745.833041:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534746112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773745.833046:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773745/real 1790773745] req@ffff9c4ef73c1c00 x1877759534745344/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773745.833048:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773745.833053:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773745.833055:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773745.833063:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2741 00000100:00080000:3.0:1790773745.833069:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773745.833074:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773745.833078:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534745344/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773745.833081:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773745.833097:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534746240/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773745.833104:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773745.833111:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773745.833116:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534745344:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773745.833118:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773745.833125:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773745.833127:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773745.833129:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534745472:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773745.833133:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2741 00000100:00080000:3.0:1790773745.833139:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773745.833145:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773745.833151:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:1.0:1790773745.833165:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773745.833180:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534746368/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773745.833182:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773745/real 1790773745] req@ffff9c4ef73c2300 x1877759534745472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773745.833205:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534745472/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773745.833223:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773745.833226:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773745.833229:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534745472:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773745.833252:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534745600:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773745.833269:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773745.833279:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773745/real 1790773745] req@ffff9c4ef73c0e00 x1877759534745600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773745.833289:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534745600/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773745.833302:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773745.833313:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773745.833316:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534745600:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773745.833324:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534745728:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773745.833340:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773745.833348:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773745/real 1790773745] req@ffff9c4ef73c0a80 x1877759534745728/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773745.833358:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534745728/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773745.833367:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773745.833370:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773745.833374:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534745728:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773745.833379:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534745856:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773745.833394:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773745.833404:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773745/real 1790773745] req@ffff9c4ef73c3800 x1877759534745856/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773745.833414:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534745856/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773745.833423:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773745.833425:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773745.833429:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534745856:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773745.833434:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534745984:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773745.833452:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773745.833463:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773745/real 1790773745] req@ffff9c4ef73c2d80 x1877759534745984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773745.833483:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534745984/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773745.833494:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773745.833496:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773745.833498:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534745984:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773745.833502:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534746112:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773745.833516:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773745.833523:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773745/real 1790773745] req@ffff9c4ef73c2a00 x1877759534746112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773745.833529:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534746112/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773745.833553:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773745.833556:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773745.833558:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534746112:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773745.833562:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534746240:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773745.833575:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773745.833582:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773745/real 1790773745] req@ffff9c4ef73c0700 x1877759534746240/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773745.833589:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534746240/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773745.833596:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773745.833597:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773745.833599:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534746240:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773745.833602:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534746368:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773745.833611:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773745.833616:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773745/real 1790773745] req@ffff9c4ef73c1500 x1877759534746368/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773745.833623:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534746368/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773745.833629:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773745.833630:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773745.833632:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534746368:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773745.896743:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773745.896743:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773745.896752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773745.896821:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773745.960705:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773745.960743:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773745.960788:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773746.088744:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773746.088756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773746.152754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773746.216751:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773746.280757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773746.280757:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773746.280775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773746.344761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773746.408747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773746.408768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773746.408846:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773746.536753:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773746.536753:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773746.536797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773746.536849:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773746.600736:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773746.600737:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773746.600748:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773746.600776:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773746.600796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773746.600798:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773746.728774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773746.792752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773746.856751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773746.920810:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773746.920811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773746.920814:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773746.920814:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773746.984748:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773746.984806:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773746.984828:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773747.112728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773747.112766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773747.176768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773747.240788:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773747.304751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773747.304761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773747.304761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773747.368732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773747.432746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773747.432747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773747.432757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773747.560721:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773747.560728:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773747.560763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773747.560765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773747.624801:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773747.624802:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773747.624804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773747.624804:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773747.624847:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773747.624875:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773747.752785:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773747.816739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773747.880794:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773747.944753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773747.944766:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773747.944767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773747.944768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773748.008778:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773748.008786:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773748.008896:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773748.136767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773748.136767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773748.200734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773748.264724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773748.328750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773748.328751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773748.328775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773748.392773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773748.456757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773748.456775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773748.456823:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773748.584774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773748.584775:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773748.584781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773748.585551:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773748.648730:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773748.648730:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773748.648732:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773748.648763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773748.648767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773748.648783:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773748.776750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773748.840775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773748.904766:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773748.968752:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773748.968752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773748.968752:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773748.968869:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773749.032759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773749.032848:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773749.032883:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773749.160743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773749.160758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773749.224782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773749.288755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773749.352761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773749.352761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773749.353168:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773749.416755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773749.480764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773749.480764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773749.481508:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773749.608755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773749.608755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773749.608755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773749.608755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773749.672755:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773749.672755:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773749.672762:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773749.672830:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773749.672833:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773749.672886:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773749.800772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773749.864735:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773749.928775:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773749.992754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773749.992755:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773749.992755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773749.992779:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773750.056761:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773750.057114:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773750.057122:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773750.184734:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773750.184737:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773750.248767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773750.312759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773750.376775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773750.376872:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773750.377717:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773750.440758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773750.504746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773750.504746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773750.504786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773750.632770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773750.632770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773750.632770:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773750.632779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773750.696793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773750.696795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773750.696930:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773750.697230:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773750.697311:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773750.697699:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773750.824767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773750.888753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773750.952767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773750.952800:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773750.952808:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773750.952813:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773750.952825:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2746 00000100:00080000:0.0:1790773750.952836:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773750.952847:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773750.952889:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534746496/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773750.952926:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773750.952933:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773750.952935:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773750.952940:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2746 00000100:00080000:0.0:1790773750.952944:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773750.952950:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773750.952961:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773750.952973:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534746496:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773750.952983:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534746624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773750.952997:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773750.953001:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773750.953004:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773750.953010:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2746 00000100:00080000:0.0:1790773750.953026:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773750.953027:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773750.953039:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773750.953046:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773750.953063:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773750/real 1790773750] req@ffff9c4ec7a7f480 x1877759534746496/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773750.953070:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534746752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773750.953084:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773750.953084:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534746496/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773750.953096:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773750.953103:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773750.953105:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773750.953111:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2746 00000100:00080000:1.0:1790773750.953113:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773750.953120:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773750.953121:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534746496:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773750.953134:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773750.953140:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773750.953146:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534746624:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773750.953158:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534746880/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773750.953171:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773750.953172:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773750.953178:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773750.953181:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773750.953188:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773750/real 1790773750] req@ffff9c4ec7a7df80 x1877759534746624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773750.953191:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2746 00000100:00080000:0.0:1790773750.953200:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773750.953210:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534746624/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773750.953214:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773750.953226:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:1.0:1790773750.953237:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773750.953245:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773750.953246:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534747008/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773750.953251:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534746624:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773750.953263:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534746752:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773750.953268:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773750.953273:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773750.953276:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773750.953283:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2746 00000800:00000100:1.0:1790773750.953285:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773750.953293:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773750.953300:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773750/real 1790773750] req@ffff9c4ec7a7ed80 x1877759534746752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773750.953306:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773750.953312:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534746752/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773750.953314:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773750.953320:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773750.953324:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773750.953327:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534746752:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773750.953337:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534747136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773750.953341:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534746880:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773750.953359:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773750.953360:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773750.953367:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773750.953370:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773750.953371:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773750/real 1790773750] req@ffff9c4ec7a7d880 x1877759534746880/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773750.953382:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2746 00000100:00080000:0.0:1790773750.953388:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773750.953395:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534746880/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773750.953396:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773750.953405:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773750.953407:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773750.953413:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773750.953416:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534746880:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773750.953421:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534747008:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773750.953445:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534747264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773750.953455:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773750.953459:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773750.953464:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773750.953466:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773750.953472:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773750/real 1790773750] req@ffff9c4ec7a7c380 x1877759534747008/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773750.953474:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2746 00000100:00080000:0.0:1790773750.953482:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773750.953490:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773750.953494:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534747008/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773750.953498:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773750.953503:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773750.953507:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773750.953510:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534747008:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773750.953516:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534747392/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773750.953524:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534747136:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773750.953531:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773750.953536:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773750.953538:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773750.953544:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2746 00000800:00000100:1.0:1790773750.953545:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773750.953552:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773750.953557:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773750/real 1790773750] req@ffff9c4ec7a7ca80 x1877759534747136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773750.953564:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773750.953570:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534747136/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773750.953574:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773750.953583:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773750.953589:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773750.953595:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534747136:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773750.953598:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534747520/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773750.953605:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534747264:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773750.953634:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773750.953646:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773750/real 1790773750] req@ffff9c4ec7a7fb80 x1877759534747264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773750.953691:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534747264/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773750.953699:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773750.953702:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773750.953705:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534747264:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773750.953717:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534747392:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773750.953736:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773750.953747:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773750/real 1790773750] req@ffff9c4ec7a7e300 x1877759534747392/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773750.953758:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534747392/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773750.953774:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773750.953778:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773750.953796:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534747392:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773750.953803:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534747520:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773750.953824:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773750.953835:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773750/real 1790773750] req@ffff9c4ec7a7c700 x1877759534747520/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773750.953846:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534747520/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773750.953856:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773750.953860:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773750.953863:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534747520:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773751.016758:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773751.016758:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773751.016864:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773751.016921:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773751.080755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773751.080755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773751.080797:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773751.208751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773751.208751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773751.272766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773751.336765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773751.400713:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773751.400722:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773751.400730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773751.464731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773751.528764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773751.528764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773751.528768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773751.656744:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773751.656744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773751.656744:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773751.656747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773751.720752:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773751.720752:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773751.720753:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773751.720757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773751.720779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773751.720786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773751.848750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773751.912721:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773751.976751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773752.040708:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773752.040718:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773752.040721:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773752.040756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773752.104758:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773752.104840:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773752.104880:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773752.233039:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773752.233039:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773752.296721:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773752.360770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773752.424754:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773752.424754:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773752.424760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773752.488762:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773752.552786:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773752.552786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773752.552786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773752.680759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773752.680759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773752.680780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773752.680801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773752.744749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773752.744749:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773752.744750:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773752.744881:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773752.744903:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773752.744930:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773752.872767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773752.936761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773753.000778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773753.064766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773753.064766:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773753.064766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773753.064766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773753.128788:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773753.128868:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773753.128884:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773753.256751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773753.256751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773753.320751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773753.384764:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773753.448724:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773753.448724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773753.448726:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773753.512728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773753.576756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773753.576756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773753.576756:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773753.704760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773753.704761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773753.704761:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773753.704766:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773753.768756:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773753.768757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773753.768758:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773753.768760:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773753.768788:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773753.768800:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773753.896732:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773753.960759:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773754.024778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773754.088764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773754.088764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773754.088764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773754.088877:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773754.152752:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773754.152787:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773754.152905:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773754.280775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773754.280775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773754.344749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773754.408724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773754.472776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773754.472776:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773754.472880:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773754.536752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773754.600719:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773754.600719:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773754.600739:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773754.728755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773754.728755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773754.728755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773754.728758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773754.792752:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773754.792753:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773754.792754:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773754.792758:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773754.792785:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773754.792806:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773754.920739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773754.984756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773755.048747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773755.112741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773755.112742:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773755.112742:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773755.113020:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773755.176758:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773755.176758:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773755.176833:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773755.304760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773755.304760:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773755.368744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773755.432727:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773755.496743:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773755.496743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773755.496743:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773755.560765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773755.624789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773755.624803:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773755.624884:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773755.752759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773755.752759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773755.752760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773755.752760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773755.816736:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773755.816736:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773755.816746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773755.816759:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773755.816767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773755.816775:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773755.944768:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773756.008750:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773756.072764:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773756.072776:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773756.072781:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773756.072786:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773756.072795:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2751 00000100:00080000:3.0:1790773756.072805:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773756.072815:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773756.072849:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534747648/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773756.072878:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773756.072882:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773756.072884:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773756.072888:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2751 00000100:00080000:3.0:1790773756.072892:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773756.072896:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773756.072914:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773756.072929:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534747648:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773756.072930:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534747776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773756.072947:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773756.072950:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773756.072952:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773756.072957:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2751 00000100:00080000:3.0:1790773756.072961:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773756.072966:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773756.072971:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773756.072983:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534747904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773756.072993:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773756.072997:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773756.072999:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773756.073003:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773756.073006:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2751 00000100:00080000:3.0:1790773756.073013:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773756.073019:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773756.073025:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773756.073035:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534748032/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773756.073048:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773756.073050:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773756.073052:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773756.073055:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2751 00000100:00100000:1.0:1790773756.073059:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773756/real 1790773756] req@ffff9c4ef73c0000 x1877759534747648/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773756.073060:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773756.073067:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773756.073075:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773756.073088:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534747648/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773756.073088:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534748160/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773756.073106:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773756.073110:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773756.073110:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773756.073113:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2751 00000100:00080000:3.0:1790773756.073118:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773756.073121:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773756.073122:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773756.073127:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773756.073131:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790773756.073135:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534748288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773756.073138:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534747648:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773756.073152:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773756.073156:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773756.073157:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773756.073163:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2751 00000100:00080000:3.0:1790773756.073167:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773756.073169:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534747776:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773756.073175:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773756.073182:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773756.073200:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773756.073202:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534748416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773756.073215:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773756.073219:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773756/real 1790773756] req@ffff9c4ef73c1500 x1877759534747776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773756.073220:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773756.073224:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773756.073233:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2751 00000100:00080000:3.0:1790773756.073237:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773756.073243:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534747776/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773756.073244:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773756.073251:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773756.073262:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534748544/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773756.073264:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773756.073271:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773756.073277:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773756.073278:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534747776:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773756.073282:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773756.073285:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773756.073288:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534747904:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773756.073290:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2751 00000100:00080000:3.0:1790773756.073294:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773756.073298:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773756.073301:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:1.0:1790773756.073324:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773756.073324:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534748672/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773756.073353:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773756/real 1790773756] req@ffff9c4ef73c0700 x1877759534747904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773756.073368:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534747904/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773756.073383:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773756.073386:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773756.073390:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534747904:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773756.073396:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534748032:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773756.073415:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773756.073423:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773756/real 1790773756] req@ffff9c4ef73c2a00 x1877759534748032/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773756.073431:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534748032/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773756.073437:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773756.073439:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773756.073441:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534748032:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773756.073445:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534748160:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773756.073456:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773756.073461:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773756/real 1790773756] req@ffff9c4ef73c2d80 x1877759534748160/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773756.073468:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534748160/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773756.073473:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773756.073475:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773756.073478:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534748160:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773756.073481:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534748288:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773756.073490:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773756.073495:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773756/real 1790773756] req@ffff9c4ef73c3800 x1877759534748288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773756.073502:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534748288/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773756.073513:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773756.073515:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773756.073516:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534748288:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773756.073531:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534748416:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773756.073541:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773756.073546:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773756/real 1790773756] req@ffff9c4ef73c0a80 x1877759534748416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773756.073553:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534748416/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773756.073558:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773756.073559:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773756.073561:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534748416:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773756.073564:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534748544:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773756.073573:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773756.073579:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773756/real 1790773756] req@ffff9c4ef73c0e00 x1877759534748544/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773756.073585:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534748544/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773756.073591:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773756.073593:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773756.073595:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534748544:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773756.073598:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534748672:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773756.073609:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773756.073619:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773756/real 1790773756] req@ffff9c4ef73c2300 x1877759534748672/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773756.073629:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534748672/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773756.073638:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773756.073641:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773756.073644:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534748672:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773756.136761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773756.136761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773756.136761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773756.136761:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773756.200775:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773756.200789:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773756.200802:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773756.328770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773756.328770:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773756.392762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773756.456758:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773756.520757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773756.520757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773756.520763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773756.584756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773756.648715:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773756.648742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773756.648780:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773756.776754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773756.776754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773756.776758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773756.776758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773756.840768:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773756.840768:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773756.840771:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773756.840902:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773756.840920:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773756.840930:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773756.968743:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773757.032765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773757.096756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773757.160748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773757.160749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773757.160751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773757.160843:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773757.224763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773757.224766:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773757.224793:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773757.352745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773757.352745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773757.416722:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773757.480769:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773757.544748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773757.544748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773757.544857:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773757.608755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773757.672755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773757.672756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773757.672756:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773757.800774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773757.800774:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773757.800896:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773757.800925:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773757.864785:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773757.864965:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773757.865057:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773757.865403:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773757.865424:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773757.865467:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773757.992755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773758.056828:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773758.120745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773758.184767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773758.184767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773758.184840:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773758.185762:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773758.248723:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773758.248739:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773758.248808:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773758.376755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773758.376755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773758.440845:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773758.504763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773758.568759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773758.568764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773758.568845:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773758.632745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773758.696762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773758.696762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773758.696762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773758.824768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773758.824768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773758.824883:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773758.824963:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773758.888756:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773758.888756:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773758.888756:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773758.888799:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773758.888807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773758.888822:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773759.016728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773759.080779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773759.144759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773759.208753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773759.208753:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773759.208753:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773759.208754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773759.272758:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773759.272793:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773759.273056:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773759.400736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773759.400737:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773759.464774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773759.528751:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773759.592744:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773759.592746:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773759.592753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773759.656791:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773759.720746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773759.720746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773759.720746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773759.848715:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773759.848716:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773759.848726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773759.848738:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773759.912733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773759.912741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773759.912742:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773759.912857:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773759.912869:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773759.912977:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773760.040762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773760.104746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773760.168709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773760.232756:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773760.232756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773760.232756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773760.232757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773760.296750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773760.296782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773760.296799:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773760.424750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773760.424752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773760.488781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773760.552763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773760.616716:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773760.616766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773760.616767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773760.680815:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773760.744793:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773760.744793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773760.744860:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773760.872768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773760.872781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773760.872785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773760.872785:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773760.936793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773760.936793:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773760.936806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773760.936810:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773760.936844:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773760.936847:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773761.064749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773761.128757:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773761.192810:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773761.192828:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1790773761.192834:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773761.192835:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773761.192852:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2757 00000100:00080000:2.0:1790773761.192865:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773761.192883:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773761.192938:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534748800/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773761.192969:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773761.192974:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773761.192976:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773761.192980:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2757 00000100:00080000:2.0:1790773761.192985:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773761.192992:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773761.193003:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773761.193011:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534748800:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773761.193025:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534748928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773761.193038:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773761.193042:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773761.193044:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773761.193049:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2757 00000100:00080000:2.0:1790773761.193054:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773761.193059:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773761.193064:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773761.193076:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773761.193082:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534749056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773761.193093:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773761.193097:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773761.193099:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773761.193104:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2757 00000100:00080000:2.0:1790773761.193108:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773761.193114:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773761.193119:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773761.193124:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773761/real 1790773761] req@ffff9c4efbcc1880 x1877759534748800/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773761.193135:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534749184/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773761.193141:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534748800/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773761.193150:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773761.193154:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773761.193155:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773761.193160:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773761.193160:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773761.193168:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534748800:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773761.193171:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2757 00000100:00080000:2.0:1790773761.193176:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773761.193182:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773761.193186:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773761.193193:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534748928:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773761.193199:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534749312/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773761.193211:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773761.193212:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773761.193218:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1790773761.193223:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773761/real 1790773761] req@ffff9c4efbcc2300 x1877759534748928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773761.193229:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773761.193251:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2757 00000100:00080000:2.0:1790773761.193256:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773761.193261:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773761.193267:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773761.195027:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534748928/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773761.195035:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534749440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773761.195059:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773761.195063:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773761.195065:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773761.195074:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2757 00000100:00080000:2.0:1790773761.195083:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773761.195088:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773761.195093:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773761.195093:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773761.195097:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534748928:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790773761.195103:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773761.195106:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534749056:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773761.195114:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534749568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773761.195122:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773761.195125:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773761.195127:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773761.195142:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2757 00000100:00080000:2.0:1790773761.195152:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773761.195156:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773761.195157:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790773761.195161:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773761.195170:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534749696/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773761.195178:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773761.195181:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773761.195183:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773761.195187:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2757 00000100:00080000:2.0:1790773761.195190:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773761.195192:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773761/real 1790773761] req@ffff9c4efbcc1500 x1877759534749056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773761.195194:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773761.195199:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773761.195207:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534749056/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773761.195207:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534749824/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773761.195221:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773761.195225:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773761.195228:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534749056:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773761.195246:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534749184:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773761.195276:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773761.195291:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773761/real 1790773761] req@ffff9c4efbcc0700 x1877759534749184/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773761.195304:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534749184/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773761.195315:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773761.195320:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773761.195323:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534749184:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773761.195354:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534749312:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773761.195376:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773761.195388:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773761/real 1790773761] req@ffff9c4efbcc2680 x1877759534749312/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773761.195401:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534749312/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773761.195412:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773761.195416:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773761.195421:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534749312:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773761.195428:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534749440:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773761.195447:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773761.195459:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773761/real 1790773761] req@ffff9c4efbcc3100 x1877759534749440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773761.195471:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534749440/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773761.195505:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773761.195510:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773761.195514:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534749440:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773761.195521:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534749568:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773761.195544:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773761.195557:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773761/real 1790773761] req@ffff9c4efbcc1c00 x1877759534749568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773761.195569:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534749568/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773761.195578:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773761.195582:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773761.195588:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534749568:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773761.195595:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534749696:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773761.195614:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773761.195641:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773761/real 1790773761] req@ffff9c4efbcc1f80 x1877759534749696/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773761.195673:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534749696/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773761.196770:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773761.196775:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773761.196777:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534749696:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773761.196782:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534749824:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773761.196800:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773761.196809:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773761/real 1790773761] req@ffff9c4efbcc0380 x1877759534749824/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773761.196816:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534749824/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773761.196825:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773761.196828:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773761.196831:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534749824:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773761.256754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773761.256754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773761.256770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773761.257713:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773761.320732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773761.320744:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773761.320794:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773761.448742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773761.448746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773761.512767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773761.577055:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773761.640739:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773761.640775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773761.640824:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773761.704766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773761.768792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773761.768857:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773761.769906:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773761.896780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773761.896780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773761.896780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773761.896803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773761.960778:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773761.960789:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773761.960811:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773761.960823:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773761.960833:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773761.960833:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773762.088786:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773762.152735:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773762.216768:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773762.280754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773762.280754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773762.280755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773762.280755:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773762.344764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773762.344904:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773762.344962:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773762.472746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773762.472746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773762.536765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773762.600724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773762.664757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773762.664757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773762.665179:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773762.728754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773762.792758:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773762.792758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773762.792758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773762.920754:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773762.920754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773762.920755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773762.920767:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773762.984775:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773762.984775:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773762.984788:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773762.984803:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773762.984813:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773762.984815:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773763.112763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773763.176744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773763.240745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773763.304752:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773763.304752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773763.304752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773763.304752:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773763.368709:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773763.368752:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773763.368773:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773763.496753:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773763.496758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773763.560769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773763.624729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773763.688730:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773763.688743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773763.688764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773763.752733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773763.816712:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773763.816713:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773763.816716:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773763.944748:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773763.944748:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773763.944748:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773763.944749:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773764.008740:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773764.008740:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773764.008740:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773764.008742:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773764.008773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773764.008774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773764.136757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773764.200734:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773764.264721:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773764.328755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773764.328755:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773764.328756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773764.328756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773764.392767:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773764.392785:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773764.392821:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773764.520756:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773764.520756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773764.584791:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773764.648753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773764.712762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773764.712762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773764.712762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773764.776769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773764.840764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773764.840765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773764.840765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773764.968721:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773764.968721:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773764.968726:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773764.968728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773765.032757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773765.032757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773765.032758:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773765.032760:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773765.032791:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773765.032805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773765.160838:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773765.224753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773765.288759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773765.352738:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773765.352738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773765.352742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773765.352747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773765.416748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773765.416774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773765.416818:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773765.544729:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773765.544735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773765.608744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773765.672734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773765.736762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773765.736762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773765.736762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773765.800736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773765.864756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773765.864756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773765.864756:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773765.992756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773765.992756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773765.992756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773765.992756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773766.056730:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773766.056730:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773766.056730:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773766.056731:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773766.056760:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773766.056761:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773766.184777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773766.248793:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773766.312798:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773766.312826:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773766.312837:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773766.312843:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773766.312854:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2762 00000100:00080000:0.0:1790773766.312867:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773766.312879:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773766.313932:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534749952/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773766.313961:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773766.313965:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773766.313967:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773766.313971:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2762 00000100:00080000:0.0:1790773766.313975:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773766.313982:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773766.313993:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773766.314006:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534749952:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773766.314008:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534750080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773766.314021:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773766.314024:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773766.314026:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773766.314030:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2762 00000100:00080000:0.0:1790773766.314033:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773766.314038:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773766.314043:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773766.314053:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534750208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773766.314062:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773766.314067:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773766.314068:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773766.314073:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2762 00000100:00080000:0.0:1790773766.314077:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773766.314082:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773766.314084:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773766.314090:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773766.314100:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534750336/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773766.314110:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773766.314113:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773766.314116:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773766.314120:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2762 00000100:00080000:0.0:1790773766.314123:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773766.314128:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773766.314147:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773766.314158:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773766/real 1790773766] req@ffff9c4ec7a7d180 x1877759534749952/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773766.314160:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534750464/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773766.314186:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773766.314190:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773766.314192:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773766.314193:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534749952/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773766.314201:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2762 00000100:00080000:0.0:1790773766.314207:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773766.314207:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773766.314214:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773766.314217:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773766.314220:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534749952:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773766.314224:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773766.314235:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534750592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773766.314242:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534750080:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773766.314251:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773766.314255:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773766.314258:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773766.314263:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2762 00000800:00000100:1.0:1790773766.314267:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773766.314268:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773766.314275:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773766.314280:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773766/real 1790773766] req@ffff9c4ec7a7c700 x1877759534750080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773766.314281:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773766.314299:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534750080/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773766.314300:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534750720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773766.314320:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773766.314324:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773766.314325:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773766.314326:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773766.314334:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773766.314335:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2762 00000100:00080000:0.0:1790773766.314340:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773766.314342:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534750080:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773766.314347:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773766.314353:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773766.314353:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534750208:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773766.314365:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534750848/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773766.314375:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773766.314379:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773766.314381:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773766.314386:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2762 00000800:00000100:1.0:1790773766.314387:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773766.314392:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773766.314399:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773766.314401:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773766/real 1790773766] req@ffff9c4ec7a7e300 x1877759534750208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773766.314405:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773766.314416:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534750208/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773766.314421:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534750976/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773766.314425:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773766.314431:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773766.314438:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534750208:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773766.314443:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534750336:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773766.314456:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773766.314463:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773766/real 1790773766] req@ffff9c4ec7a7fb80 x1877759534750336/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773766.314470:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534750336/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773766.314477:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773766.314479:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773766.314481:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534750336:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773766.314484:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534750464:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773766.314494:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773766.314504:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773766/real 1790773766] req@ffff9c4ec7a7ca80 x1877759534750464/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773766.314511:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534750464/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773766.314522:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773766.314524:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773766.314526:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534750464:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773766.314537:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534750592:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773766.314548:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773766.314553:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773766/real 1790773766] req@ffff9c4ec7a7c380 x1877759534750592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773766.314565:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534750592/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773766.314579:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773766.314581:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773766.314583:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534750592:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773766.314587:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534750720:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773766.314598:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773766.314603:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773766/real 1790773766] req@ffff9c4ec7a7d880 x1877759534750720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773766.314610:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534750720/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773766.314616:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773766.314618:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773766.314621:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534750720:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773766.314624:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534750848:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773766.314633:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773766.314638:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773766/real 1790773766] req@ffff9c4ec7a7ed80 x1877759534750848/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773766.314644:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534750848/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773766.314667:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773766.314669:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773766.314671:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534750848:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773766.314675:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534750976:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773766.314687:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773766.314693:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773766/real 1790773766] req@ffff9c4ec7a7df80 x1877759534750976/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773766.314700:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534750976/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773766.314706:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773766.314708:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773766.314710:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534750976:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773766.376767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773766.376767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773766.376771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773766.376778:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773766.440761:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773766.440832:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773766.440835:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773766.568730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773766.568730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773766.632758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773766.696786:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773766.760765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773766.760765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773766.760774:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773766.824737:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773766.888752:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773766.888753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773766.888753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773767.016795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773767.016798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773767.016799:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773767.016801:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773767.080733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773767.080746:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773767.080747:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773767.080757:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773767.080779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773767.080784:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773767.208780:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773767.272780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773767.336761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773767.400771:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773767.400771:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773767.400791:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773767.400792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773767.464783:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773767.464783:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773767.464817:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773767.592769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773767.592770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773767.656765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773767.720738:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773767.784763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773767.784763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773767.784763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773767.848750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773767.912764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773767.912764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773767.912785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773768.040760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773768.040760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773768.040772:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773768.040798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773768.104727:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773768.104732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773768.104739:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773768.104740:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773768.104759:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773768.104765:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773768.232726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773768.296770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773768.360752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773768.424741:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773768.424741:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773768.424741:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773768.424743:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773768.488741:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773768.488742:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773768.488812:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773768.616707:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773768.616727:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773768.680776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773768.744757:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773768.808760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773768.808760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773768.808760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773768.872724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773768.936740:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773768.936740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773768.936740:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773769.064759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773769.064759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773769.064759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773769.064759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773769.128734:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773769.128734:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773769.128734:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773769.128738:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773769.128764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773769.128767:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773769.256735:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773769.320742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773769.384706:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773769.448739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773769.448739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773769.448740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773769.448765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773769.512722:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773769.512776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773769.512793:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773769.640723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773769.640723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773769.704723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773769.768716:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773769.832755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773769.832755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773769.832756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773769.896758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773769.960747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773769.960747:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773769.960748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773770.088780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773770.088780:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773770.088780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773770.088823:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773770.152753:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773770.152753:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773770.152808:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773770.152888:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773770.152900:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773770.152968:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773770.280755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773770.344793:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773770.408725:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773770.472783:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773770.472783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773770.472787:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773770.472791:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773770.536789:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773770.536832:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773770.536881:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773770.664735:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773770.664756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773770.728791:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773770.792733:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773770.856767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773770.856770:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773770.856770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773770.920752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773770.984762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773770.984767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773770.984772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773771.112766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773771.112773:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773771.112779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773771.112779:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773771.176767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773771.176780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773771.176781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773771.176781:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773771.176815:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773771.176840:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773771.304729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773771.368782:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773771.432757:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773771.432761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773771.432767:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773771.432772:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773771.432782:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2767 00000100:00080000:0.0:1790773771.432793:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773771.432804:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773771.432836:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534751104/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773771.432857:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773771.432859:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773771.432861:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773771.432863:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2767 00000100:00080000:0.0:1790773771.432866:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773771.432870:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773771.432877:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773771.432886:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534751232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773771.432893:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773771.432895:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773771.432897:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773771.432900:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2767 00000100:00080000:0.0:1790773771.432903:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773771.432908:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534751104:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773771.432914:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773771.432922:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773771.432964:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534751360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773771.432976:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773771.432980:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773771.432983:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773771.432987:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2767 00000100:00080000:0.0:1790773771.432992:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773771.432997:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773771.433000:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773771.433007:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773771.433019:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534751488/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773771.433030:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773771.433047:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773771.433049:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773771.433053:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773771/real 1790773771] req@ffff9c4ec7a7f480 x1877759534751104/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773771.433057:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2767 00000100:00080000:0.0:1790773771.433064:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773771.433073:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773771.433078:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773771.433081:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534751104/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773771.433093:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534751616/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773771.433096:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773771.433105:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773771.433110:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773771.433113:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534751104:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773771.433115:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773771.433120:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773771.433126:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534751232:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773771.433129:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2767 00000100:00080000:0.0:1790773771.433136:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773771.433141:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773771.433147:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773771.433158:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773771.433161:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534751744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773771.433172:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773771.433174:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773771/real 1790773771] req@ffff9c4ec7a7df80 x1877759534751232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773771.433177:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773771.433181:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773771.433191:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2767 00000100:00080000:0.0:1790773771.433196:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773771.433198:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534751232/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773771.433204:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773771.433210:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773771.433226:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534751872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773771.433231:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773771.433237:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773771.433239:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773771.433243:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1790773771.433248:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534751232:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773771.433249:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773771.433254:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2767 00000100:00080000:0.0:1790773771.433258:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773771.433264:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773771.433269:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773771.433280:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534752000/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773771.433285:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534751360:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773771.433297:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773771.433301:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773771.433303:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773771.433308:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2767 00000100:00080000:0.0:1790773771.433313:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773771.433317:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773771.433322:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773771.433329:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773771.433329:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773771/real 1790773771] req@ffff9c4ec7a7ed80 x1877759534751360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773771.433345:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534752128/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773771.433347:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534751360/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773771.433367:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773771.433371:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773771.433373:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534751360:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773771.433378:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534751488:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773771.433392:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773771.433398:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773771/real 1790773771] req@ffff9c4ec7a7d880 x1877759534751488/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773771.433405:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534751488/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773771.433410:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773771.433412:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773771.433414:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534751488:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773771.433417:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534751616:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773771.433426:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773771.433431:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773771/real 1790773771] req@ffff9c4ec7a7c380 x1877759534751616/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773771.433437:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534751616/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773771.433447:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773771.433449:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773771.433451:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534751616:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773771.433454:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534751744:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773771.433462:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773771.433467:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773771/real 1790773771] req@ffff9c4ec7a7ca80 x1877759534751744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773771.433473:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534751744/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773771.433484:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773771.433486:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773771.433488:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534751744:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773771.433491:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534751872:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773771.433499:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773771.433505:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773771/real 1790773771] req@ffff9c4ec7a7fb80 x1877759534751872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773771.433511:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534751872/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773771.433516:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773771.433518:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773771.433520:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534751872:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773771.433531:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534752000:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773771.433540:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773771.433545:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773771/real 1790773771] req@ffff9c4ec7a7e300 x1877759534752000/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773771.433552:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534752000/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773771.433558:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773771.433560:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773771.433562:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534752000:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773771.433565:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534752128:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773771.433573:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773771.433578:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773771/real 1790773771] req@ffff9c4ec7a7c700 x1877759534752128/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773771.433585:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534752128/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773771.433590:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773771.433592:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773771.433594:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534752128:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773771.496754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773771.496754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773771.496754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773771.496754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773771.560732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773771.560775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773771.560797:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773771.688704:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773771.688722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773771.752714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773771.816700:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773771.880753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773771.880753:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773771.880753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773771.944746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773772.008730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773772.008730:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773772.008730:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773772.136749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773772.136773:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773772.136783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773772.136784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773772.200733:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773772.200743:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773772.200756:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773772.200776:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773772.200781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773772.200806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773772.328737:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773772.392765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773772.456757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773772.520756:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773772.520756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773772.520756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773772.520760:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773772.584796:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773772.584796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773772.584865:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773772.712729:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773772.712729:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773772.776746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773772.840715:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773772.904723:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773772.904728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773772.904729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773772.968731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773773.032747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773773.032747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773773.032747:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773773.160749:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773773.160750:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773773.160750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773773.160750:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773773.224737:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773773.224738:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773773.224738:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773773.224738:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773773.224763:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773773.224770:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773773.352732:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773773.416732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773773.480763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773773.544744:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773773.544744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773773.544749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773773.544851:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773773.608751:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773773.608752:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773773.608804:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773773.736743:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773773.736747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773773.800714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773773.864743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773773.928742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773773.928742:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773773.928742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773773.992759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773774.056775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773774.056776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773774.056904:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773774.184767:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773774.184768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773774.184768:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773774.184774:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773774.248746:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773774.248746:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773774.248746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773774.248747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773774.248779:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773774.248785:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773774.376719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773774.440741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773774.504705:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773774.568723:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773774.568726:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773774.568735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773774.568741:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773774.632737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773774.632738:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:3.0:1790773774.632840:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773774.760739:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773774.760739:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773774.824742:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773774.888734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773774.952718:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773774.952718:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773774.952719:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773775.016732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773775.080750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773775.080750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773775.080750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773775.208746:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773775.208746:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773775.208747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773775.208747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773775.272705:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773775.272725:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773775.272736:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773775.272738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773775.272748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773775.272761:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773775.400747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773775.464771:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773775.528735:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773775.592754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773775.592754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773775.592754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773775.592754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00010000:00010000:0.0:1790773775.656883:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773775.657097:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773775.657124:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773775.784741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773775.784742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773775.848752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773775.912754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773775.976756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773775.976756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773775.976756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773776.040725:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773776.104723:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773776.104725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773776.104731:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773776.232804:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773776.232805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773776.232805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773776.232863:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773776.296787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773776.296788:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773776.296960:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773776.296983:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773776.297035:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773776.297050:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773776.424740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773776.488780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773776.552770:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773776.552803:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773776.552808:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773776.552812:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773776.552822:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2772 00000100:00080000:2.0:1790773776.552832:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773776.552843:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773776.552877:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534752256/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773776.552905:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773776.552908:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773776.552910:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773776.552914:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2772 00000100:00080000:2.0:1790773776.552917:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773776.552922:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773776.552930:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773776.552956:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534752384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773776.552959:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534752256:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773776.552974:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773776.552978:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773776.552981:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773776.552985:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2772 00000100:00080000:2.0:1790773776.552989:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773776.552994:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773776.552999:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773776.553011:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534752512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773776.553014:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773776.553024:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773776.553028:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773776.553030:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773776.553034:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2772 00000100:00080000:2.0:1790773776.553038:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773776.553043:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773776.553047:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773776/real 1790773776] req@ffff9c4efbcc0a80 x1877759534752256/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773776.553048:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773776.553063:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534752640/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773776.553067:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534752256/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773776.553082:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773776.553086:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773776.553087:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773776.553089:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773776.553094:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773776.553097:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2772 00000100:00080000:2.0:1790773776.553102:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773776.553103:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534752256:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773776.553109:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773776.553114:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773776.553124:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534752768/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773776.553128:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534752384:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773776.553144:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773776.553149:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773776.553151:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773776.553155:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773776.553160:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2772 00000100:00100000:1.0:1790773776.553162:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773776/real 1790773776] req@ffff9c4efbcc2a00 x1877759534752384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773776.553164:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773776.553173:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773776.553176:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534752384/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773776.553187:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773776.553189:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773776.553193:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773776.553196:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534752384:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773776.553201:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534752896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773776.553202:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534752512:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773776.553217:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773776.553222:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773776.553224:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773776.553227:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773776.553231:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773776/real 1790773776] req@ffff9c4efbcc0380 x1877759534752512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773776.553234:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2772 00000100:00080000:2.0:1790773776.553240:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773776.553248:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773776.553254:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773776.553255:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534752512/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773776.553263:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773776.553268:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773776.553270:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534753024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773776.553271:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534752512:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773776.553283:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534752640:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773776.553287:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773776.553292:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773776.553294:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773776.553299:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2772 00000100:00080000:2.0:1790773776.553303:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773776.553308:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773776.553310:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773776.553316:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773776.553325:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773776/real 1790773776] req@ffff9c4efbcc1f80 x1877759534752640/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773776.553329:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534753152/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773776.553356:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534752640/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773776.553360:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773776.553367:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773776.553370:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773776.553376:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2772 00000100:00080000:1.0:1790773776.553377:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773776.553381:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773776.553384:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773776.553390:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534752640:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773776.553407:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534752768:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773776.553414:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773776.553420:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:1.0:1790773776.553431:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773776.553436:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534753280/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773776.553449:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773776/real 1790773776] req@ffff9c4efbcc1c00 x1877759534752768/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773776.553463:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534752768/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773776.553479:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773776.553483:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773776.553486:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534752768:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773776.553493:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534752896:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773776.553516:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773776.553527:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773776/real 1790773776] req@ffff9c4efbcc3100 x1877759534752896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773776.553539:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534752896/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773776.553554:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773776.553557:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773776.553561:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534752896:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773776.553567:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534753024:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773776.553585:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773776.553595:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773776/real 1790773776] req@ffff9c4efbcc2680 x1877759534753024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773776.553607:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534753024/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773776.553614:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773776.553616:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773776.553619:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534753024:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773776.553623:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534753152:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773776.553637:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773776.553644:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773776/real 1790773776] req@ffff9c4efbcc0700 x1877759534753152/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773776.553672:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534753152/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773776.553686:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773776.553688:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773776.553690:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534753152:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773776.553703:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534753280:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773776.553718:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773776.553728:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773776/real 1790773776] req@ffff9c4efbcc1500 x1877759534753280/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773776.553740:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534753280/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773776.553749:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773776.553752:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773776.553756:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534753280:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773776.616748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773776.616748:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773776.616749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773776.616760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773776.680778:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773776.680888:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773776.680919:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773776.808727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773776.808727:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773776.872756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773776.936743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773777.000764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773777.000764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773777.000793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773777.064752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773777.128748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773777.128748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773777.128748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773777.256748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773777.256748:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773777.256766:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773777.256783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773777.320730:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773777.320732:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773777.320774:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773777.320780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773777.320831:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773777.320836:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773777.448753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773777.512786:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773777.576740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773777.640796:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773777.640796:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773777.640807:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773777.640807:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773777.704765:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773777.704799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773777.704801:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773777.832752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773777.832752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773777.896756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773777.960745:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773778.024739:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773778.024740:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773778.024843:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773778.088754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773778.152711:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773778.152724:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773778.152730:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773778.280755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773778.280755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773778.280761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773778.280853:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773778.344753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773778.344754:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773778.344754:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773778.344781:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773778.344876:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773778.344892:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773778.472746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773778.536776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773778.600742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773778.664739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773778.664739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773778.664740:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773778.664740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773778.728786:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773778.728819:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773778.728867:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773778.856753:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773778.856753:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773778.920712:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773778.984745:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773779.048720:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773779.048720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773779.048723:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773779.112765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773779.176721:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773779.176722:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773779.176728:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773779.304739:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773779.304739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773779.304739:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773779.304831:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773779.368716:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773779.368717:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773779.368805:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773779.368819:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773779.368846:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773779.368961:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773779.496731:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773779.560765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773779.624769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773779.688752:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773779.688780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773779.688780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773779.688816:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773779.752722:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773779.752785:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773779.752912:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773779.880769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773779.880769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773779.944785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773780.008863:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773780.072749:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773780.072755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773780.072780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773780.136815:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773780.200837:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773780.200838:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773780.200839:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773780.328790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773780.328791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773780.328791:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773780.328912:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773780.392793:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773780.392793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773780.392803:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773780.392804:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773780.392839:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773780.392856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773780.520775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773780.584786:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773780.648788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773780.712798:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773780.712799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773780.712799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773780.712848:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773780.776756:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773780.776757:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773780.776866:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773780.904730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773780.904730:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773780.968783:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773781.032767:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773781.096777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773781.096777:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773781.096777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773781.160753:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773781.224737:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773781.224737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773781.224740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773781.352768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773781.352768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773781.352768:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773781.352768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773781.416728:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773781.416728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773781.416736:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773781.416738:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773781.416759:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773781.416768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773781.544792:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773781.608730:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773781.672773:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773781.672774:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773781.672783:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773781.672788:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773781.672797:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2777 00000100:00080000:3.0:1790773781.672807:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773781.672819:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773781.672877:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534753408/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773781.672903:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773781.672906:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773781.672908:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773781.672911:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2777 00000100:00080000:3.0:1790773781.672913:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773781.672917:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773781.672926:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773781.672938:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534753536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773781.672946:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773781.672949:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773781.672950:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773781.672954:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2777 00000100:00100000:1.0:1790773781.672958:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534753408:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773781.672958:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773781.672964:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773781.672968:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773781.672976:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534753664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773781.672982:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773781.672985:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773781.672986:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773781.672989:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2777 00000100:00080000:3.0:1790773781.672992:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773781.672994:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773781.672998:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773781.673005:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534753792/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773781.673011:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773781.673014:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773781.673014:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773781.673018:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2777 00000100:00080000:3.0:1790773781.673021:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773781.673024:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773781.673026:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773781.673031:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773781.673039:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534753920/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773781.673045:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773781.673048:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773781.673049:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773781.673051:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2777 00000100:00080000:3.0:1790773781.673053:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773781.673056:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773781.673059:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773781.673067:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534754048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773781.673073:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773781.673075:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773781/real 1790773781] req@ffff9c4ef73c1c00 x1877759534753408/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773781.673076:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773781.673078:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773781.673084:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2777 00000100:00080000:3.0:1790773781.673089:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773781.673094:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773781.673097:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773781.673111:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534753408/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773781.673111:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534754176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773781.673130:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773781.673135:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773781.673136:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773781.673139:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2777 00000100:00080000:1.0:1790773781.673140:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773781.673142:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773781.673145:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773781.673147:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773781.673152:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773781.673153:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534753408:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773781.673159:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534754304/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773781.673171:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773781.673173:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773781.673175:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773781.673178:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2777 00000100:00080000:3.0:1790773781.673180:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773781.673183:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773781.673185:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773781.673189:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534753536:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773781.673194:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534754432/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773781.673215:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773781.673226:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773781/real 1790773781] req@ffff9c4ef73c2300 x1877759534753536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773781.673234:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534753536/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773781.673254:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773781.673257:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773781.673259:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534753536:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773781.673263:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534753664:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773781.673277:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773781.673284:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773781/real 1790773781] req@ffff9c4ef73c0e00 x1877759534753664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773781.673291:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534753664/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773781.673297:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773781.673300:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773781.673303:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534753664:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773781.673309:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534753792:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773781.673327:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773781.673338:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773781/real 1790773781] req@ffff9c4ef73c0a80 x1877759534753792/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773781.673350:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534753792/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773781.673360:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773781.673363:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773781.673367:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534753792:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773781.673373:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534753920:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773781.673398:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773781.673407:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773781/real 1790773781] req@ffff9c4ef73c3800 x1877759534753920/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773781.673419:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534753920/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773781.673426:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773781.673428:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773781.673430:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534753920:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773781.673434:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534754048:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773781.673446:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773781.673452:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773781/real 1790773781] req@ffff9c4ef73c2d80 x1877759534754048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773781.673459:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534754048/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773781.673469:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773781.673471:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773781.673473:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534754048:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773781.673476:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534754176:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773781.673486:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773781.673492:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773781/real 1790773781] req@ffff9c4ef73c2a00 x1877759534754176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773781.673498:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534754176/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773781.673507:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773781.673509:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773781.673511:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534754176:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773781.673514:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534754304:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773781.673523:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773781.673528:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773781/real 1790773781] req@ffff9c4ef73c0700 x1877759534754304/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773781.673534:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534754304/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773781.673540:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773781.673542:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773781.673544:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534754304:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773781.673566:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534754432:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773781.673577:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773781.673584:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773781/real 1790773781] req@ffff9c4ef73c1500 x1877759534754432/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773781.673591:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534754432/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773781.673597:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773781.673599:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773781.673601:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534754432:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773781.736791:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773781.736791:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773781.736791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773781.736806:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773781.800829:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773781.800984:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773781.801011:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773781.928767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773781.928767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773781.992778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773782.056726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773782.120766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773782.120766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773782.120769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773782.184779:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773782.248764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773782.248764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773782.248764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773782.376736:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773782.376736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773782.376736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773782.376737:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773782.440731:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773782.440731:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773782.440732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773782.440737:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773782.440770:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773782.440772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773782.568773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773782.632770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773782.696736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773782.760768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773782.760768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773782.760768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773782.760768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773782.824741:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773782.824790:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773782.824884:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773782.952757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773782.952757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773783.016758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773783.080775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773783.144826:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773783.144826:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773783.144828:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773783.208740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773783.272741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773783.272741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773783.272773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773783.400799:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773783.400800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773783.400800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773783.400811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773783.464733:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773783.464734:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773783.464746:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773783.464773:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773783.464784:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773783.464787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773783.592795:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773783.656742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773783.720769:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773783.784805:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773783.784812:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773783.784813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773783.784955:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773783.848766:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773783.848819:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773783.848944:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773783.976752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773783.976754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773784.040821:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773784.104797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773784.168820:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773784.168825:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773784.168874:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773784.232800:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773784.296832:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773784.296832:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773784.296833:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773784.424743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773784.424743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773784.424743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773784.424805:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773784.488807:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773784.488808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773784.488808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773784.488809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773784.488883:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773784.488884:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773784.616755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773784.680741:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773784.744787:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773784.808769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773784.808769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773784.808769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773784.808901:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773784.872791:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773784.872792:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773784.872869:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773785.000767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773785.000782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773785.065002:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773785.128740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773785.192751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773785.192751:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773785.192752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773785.256716:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773785.320754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773785.320754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773785.320855:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773785.448755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773785.448755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773785.448755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773785.448788:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773785.512743:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773785.512781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773785.512785:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773785.512820:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773785.512882:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773785.512889:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773785.640738:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773785.704788:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773785.768743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773785.832796:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773785.832796:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773785.832800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773785.832801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773785.896792:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773785.896844:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773785.896857:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773786.024763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773786.024764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773786.088763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773786.152740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773786.216765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773786.216790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773786.216892:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773786.280797:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773786.344818:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773786.344818:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773786.344822:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773786.472750:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773786.472751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773786.472751:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773786.472754:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773786.536771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773786.536771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773786.536771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773786.536805:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773786.536917:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773786.536920:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773786.664798:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773786.728810:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773786.792752:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790773786.792753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773786.792762:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773786.792766:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773786.792775:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2782 00000100:00080000:2.0:1790773786.792783:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773786.792795:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773786.792829:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534754560/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773786.792854:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773786.792858:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773786.792860:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773786.792864:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2782 00000100:00080000:2.0:1790773786.792867:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773786.792871:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773786.792881:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773786.792893:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534754688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773786.792904:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534754560:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773786.792905:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773786.792913:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773786.792915:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773786.792920:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2782 00000100:00080000:2.0:1790773786.792924:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773786.792928:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773786.792933:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773786.792944:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534754816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773786.792951:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773786.792953:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773786.792962:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773786.792966:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2782 00000100:00080000:2.0:1790773786.792969:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773786.792973:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773786.792977:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773786.792985:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534754944/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773786.792991:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773786.792994:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773786.792995:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773786.792998:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2782 00000100:00080000:2.0:1790773786.793000:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773786.793005:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534754688:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773786.793006:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773786.793014:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773786.793026:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534755072/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773786.793040:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534754816:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773786.793042:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773786.793048:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773786.793051:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773786.793057:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534754944:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773786.793057:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2782 00000100:00080000:2.0:1790773786.793069:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773786.793078:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773786.793083:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773786.793088:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534755072:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773786.793101:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534755200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773786.793114:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773786.793117:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1790773786.793120:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534755200:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773786.793122:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773786.793130:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2782 00000100:00080000:2.0:1790773786.793136:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773786.793142:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773786.793147:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773786.793159:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534755328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773786.793174:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773786.793177:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773786.793179:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773786.793183:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2782 00000100:00080000:2.0:1790773786.793187:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773786.793192:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773786.793198:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773786.793199:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534755328:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773786.793210:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534755456/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773786.793221:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773786.793224:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773786.793226:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773786.793231:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2782 00000100:00100000:1.0:1790773786.793235:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534755456:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773786.793237:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773786.793245:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773786.793250:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773786.793264:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534755584/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773786.793292:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534755584:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773786.856745:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773786.856750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773786.856752:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773786.856850:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773786.920781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773786.920783:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773786.921376:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773787.048757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773787.048757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773787.112753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773787.176724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773787.240770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773787.240771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773787.240771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773787.304742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773787.368755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773787.368755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773787.368755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773787.496790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773787.496790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773787.496790:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773787.496802:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773787.560792:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773787.560806:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773787.560824:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773787.560829:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773787.560878:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773787.560894:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773787.688798:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773787.752825:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773787.816771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773787.880765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773787.880765:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773787.880765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773787.880772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773787.944794:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773787.944806:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773787.944891:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773788.072726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773788.072727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773788.136800:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773788.200789:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773788.264776:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773788.264781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773788.264790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773788.328740:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773788.392786:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773788.392786:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773788.392786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773788.520778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773788.520778:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773788.520778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773788.520779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773788.584783:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773788.584784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773788.584784:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773788.584788:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773788.584832:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773788.584851:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773788.712797:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773788.776738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773788.840759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773788.904769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773788.904795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773788.904798:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773788.904802:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773788.968784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773788.968808:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773788.968932:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773789.096785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773789.096791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773789.160785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773789.224739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773789.288780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773789.288780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773789.288789:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773789.352781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773789.416768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773789.416776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773789.416776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773789.544771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773789.544774:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773789.544779:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773789.544779:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773789.608734:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773789.608775:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773789.608776:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773789.608777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773789.608799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773789.608808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773789.736747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773789.800816:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773789.864753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:0.0:1790773789.865014:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:0.0:1790773789.865078:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1790773789.865100:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790773789.865160:0:2383:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773789.865227:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773786/real 1790773789] req@ffff9c4efbcc2300 x1877759534754560/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1790773789.865242:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790773789.865263:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:1.0:1790773789.865286:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534754560/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1790773789.865287:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790773789.865304:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:1.0:1790773789.865305:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773789.865318:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773789.865326:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534754560:192.168.204.150@tcp:250:kworker.0 00000800:00000100:0.0:1790773789.865327:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:0.0:1790773789.865352:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:1.0:1790773789.865369:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773786/real 1790773789] req@ffff9c4efbcc0380 x1877759534755584/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1790773789.865376:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:1.0:1790773789.865404:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534755584/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773789.865437:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773789.865442:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773789.865446:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534755584:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773789.865454:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773786/real 1790773789] req@ffff9c4efbcc1f80 x1877759534755456/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773789.865467:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534755456/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773789.865478:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773789.865481:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773789.865485:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534755456:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773789.865493:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773786/real 1790773789] req@ffff9c4efbcc1c00 x1877759534755328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773789.865505:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534755328/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773789.865511:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773789.865513:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773789.865516:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534755328:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773789.865522:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773786/real 1790773789] req@ffff9c4efbcc3100 x1877759534755200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773789.865534:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534755200/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773789.865550:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773789.865554:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773789.865557:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534755200:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773789.865565:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773786/real 1790773789] req@ffff9c4efbcc2680 x1877759534755072/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773789.865575:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534755072/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773789.865584:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773789.865587:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773789.865590:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534755072:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773789.865597:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773786/real 1790773789] req@ffff9c4efbcc1880 x1877759534754688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773789.865610:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534754688/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773789.865625:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773789.865629:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773789.865632:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534754688:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773789.865639:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773786/real 1790773789] req@ffff9c4efbcc1500 x1877759534754816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773789.865689:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534754816/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773789.865703:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773789.865707:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773789.865711:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534754816:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773789.865725:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773786/real 1790773789] req@ffff9c4efbcc0700 x1877759534754944/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773789.865741:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534754944/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773789.865751:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773789.865754:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773789.865758:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534754944:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773789.928751:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773789.928751:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773789.928751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773789.928796:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773789.992796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773789.992811:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773789.992937:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773790.120779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773790.120785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773790.184809:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773790.248763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773790.312747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773790.312747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773790.312747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773790.376734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773790.440723:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773790.440727:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773790.440731:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773790.568762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773790.568762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773790.568762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773790.568762:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773790.632746:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773790.632746:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773790.632749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773790.632860:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773790.632870:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773790.632916:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773790.760749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773790.824728:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773790.888744:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773790.952764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773790.952764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773790.952765:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773790.952780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773791.016770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773791.016797:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773791.016984:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773791.144714:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773791.144731:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773791.208727:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773791.272740:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773791.336789:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773791.336797:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773791.336809:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773791.400805:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773791.464745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773791.464745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773791.464756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773791.592813:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773791.592814:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773791.592819:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773791.592831:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773791.656753:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773791.656754:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773791.656766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773791.656769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773791.656804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773791.656807:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773791.784749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773791.848739:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773791.912753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773791.912778:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773791.912786:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773791.912791:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773791.912808:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2787 00000100:00080000:2.0:1790773791.912821:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773791.912832:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773791.912871:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534755712/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773791.912909:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773791.912917:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773791.912920:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773791.912925:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2787 00000100:00080000:2.0:1790773791.912929:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773791.912934:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773791.912947:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773791.912963:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534755840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773791.912974:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773791.912977:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773791.912979:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773791.912984:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2787 00000100:00080000:2.0:1790773791.912988:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773791.912991:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534755712:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773791.912996:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773791.913003:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773791.913017:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534755968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773791.913026:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773791.913031:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773791.913033:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773791.913038:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2787 00000100:00080000:2.0:1790773791.913043:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773791.913048:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773791.913053:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773791.913064:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534756096/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773791.913073:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773791.913076:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773791.913078:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773791.913081:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773791.913085:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2787 00000100:00080000:2.0:1790773791.913089:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773791.913093:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773791.913098:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773791.913108:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534756224/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773791.913116:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773791.913119:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773791.913121:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773791.913126:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2787 00000100:00100000:1.0:1790773791.913133:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773791/real 1790773791] req@ffff9c4efbcc2a00 x1877759534755712/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773791.913135:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773791.913143:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773791.913151:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773791.913161:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534755712/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773791.913165:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534756352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773791.913212:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773791.913216:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773791.913216:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773791.913220:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773791.913225:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773791.913228:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2787 00000100:00100000:1.0:1790773791.913234:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534755712:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773791.913234:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773791.913242:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773791.913247:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773791.913258:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534756480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773791.913268:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773791.913271:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534755840:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773791.913272:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773791.913275:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773791.913282:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2787 00000100:00080000:2.0:1790773791.913286:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773791.913291:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773791.913296:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:1.0:1790773791.913305:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773791.913309:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534756608/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773791.913318:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773791.913323:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773791.913324:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773791/real 1790773791] req@ffff9c4efbcc0a80 x1877759534755840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773791.913325:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773791.913333:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2787 00000100:00080000:2.0:1790773791.913339:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773791.913344:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773791.913348:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773791.913360:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534756736/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773791.913363:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534755840/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773791.913398:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773791.913402:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773791.913407:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534755840:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773791.913415:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534755968:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773791.913436:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773791.913447:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773791/real 1790773791] req@ffff9c4efbcc0700 x1877759534755968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773791.913460:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534755968/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773791.913468:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773791.913472:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773791.913475:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534755968:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773791.913482:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534756096:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773791.913502:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773791.913513:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773791/real 1790773791] req@ffff9c4efbcc1500 x1877759534756096/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773791.913530:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534756096/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773791.913540:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773791.913543:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773791.913547:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534756096:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773791.913552:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534756224:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773791.913569:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773791.913579:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773791/real 1790773791] req@ffff9c4efbcc1880 x1877759534756224/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773791.913588:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534756224/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773791.913597:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773791.913600:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773791.913603:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534756224:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773791.913608:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534756352:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773791.913625:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773791.913634:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773791/real 1790773791] req@ffff9c4efbcc2680 x1877759534756352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773791.913685:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534756352/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773791.913701:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773791.913704:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773791.913707:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534756352:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773791.913729:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534756480:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773791.913749:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773791.913760:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773791/real 1790773791] req@ffff9c4efbcc3100 x1877759534756480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773791.913770:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534756480/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773791.913778:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773791.913781:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773791.913784:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534756480:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773791.913789:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534756608:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773791.913807:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773791.913818:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773791/real 1790773791] req@ffff9c4efbcc1c00 x1877759534756608/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773791.913828:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534756608/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773791.913844:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773791.913847:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773791.913850:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534756608:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773791.913856:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534756736:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773791.913881:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773791.913891:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773791/real 1790773791] req@ffff9c4efbcc1f80 x1877759534756736/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773791.913903:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534756736/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773791.913912:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773791.913916:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773791.913920:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534756736:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773791.976729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773791.976732:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773791.976734:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773791.976760:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773792.040755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773792.040805:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773792.040862:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773792.168785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773792.168787:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773792.232856:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773792.296750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773792.360734:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773792.360734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773792.360757:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773792.424734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773792.488750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773792.488751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773792.488751:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773792.616780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773792.616780:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773792.616913:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773792.616993:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773792.680727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773792.680727:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773792.680728:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773792.680766:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773792.680776:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773792.680784:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773792.808741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773792.872746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773792.936738:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773793.000761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773793.000761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773793.000761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773793.000763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773793.064759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773793.064771:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773793.064814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773793.192723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773793.192725:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773793.256751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773793.320748:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773793.384742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773793.384742:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773793.384742:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773793.448764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773793.512764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773793.512768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773793.512768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773793.640810:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773793.640812:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773793.640814:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773793.640821:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773793.704748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773793.704779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773793.704810:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773793.704812:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773793.704932:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773793.704973:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773793.832761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773793.896793:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773793.960736:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773794.024729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773794.024752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773794.024752:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773794.024752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773794.088753:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773794.088765:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773794.088805:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773794.216809:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773794.216810:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773794.280777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773794.344781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773794.408720:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773794.408720:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773794.408721:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773794.472707:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773794.536770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773794.536770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773794.536770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773794.664745:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773794.664745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773794.664750:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773794.664786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773794.728745:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773794.728756:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773794.728770:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773794.728787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773794.728817:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773794.728823:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773794.856734:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773794.920776:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773794.984747:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773795.048761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773795.048771:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773795.048782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773795.048873:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773795.112730:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773795.112780:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773795.112800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773795.240755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773795.240755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773795.304753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773795.368773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773795.432754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773795.432754:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773795.432754:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773795.496744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773795.560721:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773795.560721:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773795.560731:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773795.688755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773795.688755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773795.688756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773795.688763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773795.752747:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773795.752747:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773795.752750:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773795.752778:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773795.752893:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773795.752945:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773795.880736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773795.944744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773796.008696:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773796.072769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773796.072770:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773796.072770:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773796.072799:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773796.136750:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773796.136775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773796.136787:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773796.264742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773796.264742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773796.328723:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773796.392747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773796.456749:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773796.456749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773796.456749:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773796.520742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773796.584771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773796.584775:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773796.584862:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773796.712766:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773796.712767:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773796.712770:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773796.712774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773796.776763:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773796.776763:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773796.776768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773796.776771:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773796.776797:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773796.776800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773796.904738:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773796.968753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773797.032754:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773797.032786:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773797.032791:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773797.032794:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773797.032803:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2792 00000100:00080000:0.0:1790773797.032812:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773797.032823:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773797.032857:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534756864/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773797.032879:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773797.032881:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773797.032883:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773797.032885:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2792 00000100:00080000:0.0:1790773797.032888:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773797.032891:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773797.032899:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773797.032909:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534756992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773797.032916:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773797.032918:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773797.032920:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773797.032923:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2792 00000100:00080000:0.0:1790773797.032926:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773797.032928:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773797.032931:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773797.032943:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534757120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773797.032943:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534756864:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773797.032956:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773797.032958:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773797.032960:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773797.032962:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2792 00000100:00080000:0.0:1790773797.032965:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773797.032967:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773797.032971:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773797.032980:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534757248/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773797.032985:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773797.032988:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773797.032989:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773797.032992:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2792 00000100:00080000:0.0:1790773797.032995:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773797.032998:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773797.033002:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773797.033004:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773797.033014:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534757376/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773797.033020:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773797.033023:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773797.033024:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773797.033027:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2792 00000100:00080000:0.0:1790773797.033030:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773797.033034:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773797.033037:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773797/real 1790773797] req@ffff9c4ec7a7d180 x1877759534756864/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773797.033038:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773797.033049:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534757504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773797.033056:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534756864/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773797.033059:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773797.033065:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773797.033069:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773797.033076:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2792 00000100:00080000:0.0:1790773797.033080:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773797.033085:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773797.033086:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790773797.033088:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773797.033092:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773797.033096:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534756864:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773797.033098:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534757632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773797.033104:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773797.033108:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773797.033110:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773797.033115:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2792 00000100:00080000:0.0:1790773797.033117:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773797.033119:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534756992:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773797.033123:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773797.033127:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773797.033137:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534757760/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773797.033142:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773797.033152:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773797/real 1790773797] req@ffff9c4ec7a7c700 x1877759534756992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773797.033153:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773797.033158:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773797.033161:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773797.033167:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2792 00000100:00080000:0.0:1790773797.033173:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773797.033176:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773797.033180:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773797.033183:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534756992/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773797.033190:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534757888/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773797.033207:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773797.033213:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773797.033217:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534756992:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773797.033242:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534757120:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773797.033264:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773797.033275:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773797/real 1790773797] req@ffff9c4ec7a7e300 x1877759534757120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773797.033285:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534757120/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773797.033293:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773797.033296:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773797.033300:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534757120:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773797.033306:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534757248:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773797.033322:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773797.033331:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773797/real 1790773797] req@ffff9c4ec7a7fb80 x1877759534757248/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773797.033341:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534757248/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773797.033349:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773797.033352:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773797.033355:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534757248:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773797.033360:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534757376:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773797.033376:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773797.033386:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773797/real 1790773797] req@ffff9c4ec7a7ca80 x1877759534757376/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773797.033397:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534757376/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773797.033406:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773797.033409:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773797.033412:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534757376:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773797.033418:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534757504:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773797.033436:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773797.033447:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773797/real 1790773797] req@ffff9c4ec7a7c380 x1877759534757504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773797.033457:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534757504/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773797.033467:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773797.033469:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773797.033471:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534757504:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773797.033475:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534757632:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773797.033487:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773797.033493:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773797/real 1790773797] req@ffff9c4ec7a7d880 x1877759534757632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773797.033500:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534757632/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773797.033513:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773797.033514:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773797.033516:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534757632:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773797.033520:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534757760:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773797.033530:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773797.033552:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773797/real 1790773797] req@ffff9c4ec7a7ed80 x1877759534757760/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773797.033564:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534757760/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773797.033581:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773797.033584:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773797.033587:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534757760:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773797.033594:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534757888:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773797.033608:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773797.033615:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773797/real 1790773797] req@ffff9c4ec7a7df80 x1877759534757888/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773797.033622:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534757888/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773797.033628:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773797.033630:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773797.033633:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534757888:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773797.096746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773797.096746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773797.096746:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773797.096844:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773797.160736:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773797.160854:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773797.160855:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773797.288743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773797.288743:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773797.352750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773797.416746:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773797.480734:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773797.480734:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773797.480739:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773797.544726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773797.608751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773797.608751:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773797.608751:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773797.736780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773797.736781:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773797.736800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773797.736816:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773797.800754:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773797.800754:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773797.800774:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773797.800780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773797.800806:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773797.800814:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773797.928729:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773797.992791:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773798.056739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773798.120749:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773798.120749:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773798.120750:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773798.120750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773798.184745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773798.184773:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773798.184826:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773798.312732:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773798.312784:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773798.376714:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773798.440741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773798.504747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773798.504747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773798.504752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773798.568718:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773798.632737:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773798.632737:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773798.632737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773798.760726:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773798.760727:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773798.760727:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773798.760727:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773798.824708:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773798.824715:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773798.824730:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773798.824730:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773798.824745:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773798.824753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773798.952741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773799.016764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773799.080742:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773799.144761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773799.144761:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773799.144761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773799.144762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773799.208712:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773799.208713:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773799.208800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773799.336742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773799.336742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773799.400763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773799.464728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773799.528735:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773799.528735:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773799.528739:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773799.592739:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773799.656754:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773799.656754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773799.656754:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773799.784765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773799.784765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773799.784765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773799.784880:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773799.848726:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773799.848727:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773799.848732:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773799.848733:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773799.848758:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773799.848759:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773799.976715:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773800.040716:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773800.104763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773800.168739:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773800.168739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773800.168739:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773800.168745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773800.232754:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773800.232821:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773800.232822:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773800.360708:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773800.360727:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773800.424752:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773800.488725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773800.552769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773800.552769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773800.552778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773800.616763:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773800.680791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773800.680791:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773800.680792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773800.808745:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773800.808745:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773800.808757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773800.808758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773800.872787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773800.872787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773800.872787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773800.872802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773800.872829:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773800.872840:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773801.000725:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773801.064773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773801.128711:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773801.192751:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773801.192751:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773801.192765:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773801.192783:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773801.256779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773801.256828:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773801.256844:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773801.384726:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773801.384726:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773801.448753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773801.512722:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773801.576710:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773801.576719:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773801.576735:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773801.640770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773801.704736:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773801.704739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773801.704757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773801.832790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773801.832790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773801.832795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773801.832907:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773801.896771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773801.896771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773801.896809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773801.896893:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773801.896893:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773801.896916:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773802.024711:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773802.088786:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773802.152791:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790773802.152798:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773802.152807:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773802.152814:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773802.152827:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2798 00000100:00080000:0.0:1790773802.152841:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773802.152855:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773802.152900:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534758016/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773802.152928:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773802.152931:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773802.152933:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773802.152936:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2798 00000100:00080000:0.0:1790773802.152938:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773802.152942:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773802.152951:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773802.152963:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534758144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773802.152972:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773802.152974:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773802.152976:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773802.152983:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534758016:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773802.152985:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2798 00000100:00080000:0.0:1790773802.152991:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773802.153002:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773802.153007:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773802.153022:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534758272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773802.153032:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773802.153037:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773802.153039:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773802.153045:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2798 00000100:00080000:0.0:1790773802.153049:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773802.153053:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773802.153056:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773802.153063:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773802.153078:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534758400/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773802.153087:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773802.153091:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773802.153093:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773802.153097:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773802/real 1790773802] req@ffff9c4ec7a7f480 x1877759534758016/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773802.153104:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2798 00000100:00080000:0.0:1790773802.153118:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773802.153125:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534758016/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773802.153128:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773802.153142:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:1.0:1790773802.153145:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773802.153151:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773802.153163:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534758016:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773802.153164:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534758528/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773802.153183:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773802.153187:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773802.153189:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773802.153191:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534758144:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773802.153199:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2798 00000100:00080000:0.0:1790773802.153206:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773802.153213:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773802.153217:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773802.153230:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534758656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773802.153238:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773802.153254:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773802/real 1790773802] req@ffff9c4ec7a7df80 x1877759534758144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773802.153310:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773802.153316:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534758144/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773802.153317:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773802.153324:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773802.153339:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2798 00000100:00080000:0.0:1790773802.153345:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773802.153348:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773802.153352:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773802.153358:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790773802.153362:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773802.153362:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534758144:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773802.153376:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534758272:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773802.153383:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534758784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773802.153391:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773802.153399:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773802.153403:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773802.153405:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773802.153412:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2798 00000100:00100000:1.0:1790773802.153415:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773802/real 1790773802] req@ffff9c4ec7a7ed80 x1877759534758272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773802.153417:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773802.153426:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773802.153434:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773802.153451:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534758272/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773802.153460:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534758912/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773802.153482:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773802.153484:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773802.153494:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773802.153496:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773802.153500:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773802.153505:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2798 00000100:00080000:0.0:1790773802.153512:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773802.153517:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534758272:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773802.153520:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773802.153532:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773802.153538:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534758400:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773802.153542:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534759040/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773802.153572:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773802.153582:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773802/real 1790773802] req@ffff9c4ec7a7d880 x1877759534758400/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773802.153591:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534758400/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773802.153601:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773802.153604:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773802.153607:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534758400:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773802.153614:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534758528:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773802.153634:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773802.153645:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773802/real 1790773802] req@ffff9c4ec7a7c380 x1877759534758528/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773802.153681:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534758528/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773802.153688:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773802.153691:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773802.153693:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534758528:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773802.153706:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534758656:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773802.153719:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773802.153726:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773802/real 1790773802] req@ffff9c4ec7a7ca80 x1877759534758656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773802.153733:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534758656/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773802.153748:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773802.153750:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773802.153752:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534758656:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773802.153756:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534758784:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773802.153764:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773802.153770:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773802/real 1790773802] req@ffff9c4ec7a7fb80 x1877759534758784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773802.153776:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534758784/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773802.153781:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773802.153783:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773802.153785:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534758784:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773802.153788:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534758912:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773802.153796:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773802.153802:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773802/real 1790773802] req@ffff9c4ec7a7e300 x1877759534758912/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773802.153808:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534758912/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773802.153814:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773802.153816:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773802.153818:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534758912:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773802.153822:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534759040:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773802.153830:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773802.153835:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773802/real 1790773802] req@ffff9c4ec7a7c700 x1877759534759040/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773802.153841:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534759040/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773802.153846:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773802.153848:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773802.153850:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534759040:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773802.216738:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773802.216742:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773802.216779:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773802.216779:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773802.280774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773802.280819:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773802.280832:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773802.408747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773802.408806:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773802.472778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773802.536738:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773802.600761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773802.600761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773802.600761:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773802.664728:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773802.728730:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773802.728731:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773802.728735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773802.856811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773802.856811:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773802.856812:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773802.856813:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773802.920736:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773802.920737:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773802.920737:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773802.920782:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773802.920786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773802.920789:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773803.048773:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773803.112774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773803.176758:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773803.240753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773803.240754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773803.240755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773803.240776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773803.304753:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773803.304801:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773803.304810:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773803.432763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773803.432763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773803.496769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773803.560783:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773803.624780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773803.624790:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773803.624793:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773803.688801:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773803.752768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773803.752768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773803.752779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773803.880805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773803.880806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773803.880809:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773803.880810:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773803.944804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773803.944804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773803.944814:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773803.944814:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773803.944863:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773803.944869:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773804.072748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773804.136801:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773804.200730:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773804.264769:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773804.264769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773804.264770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773804.264805:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773804.328755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773804.328761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773804.328829:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773804.456747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773804.456747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773804.520751:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773804.584747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773804.648741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773804.648741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773804.648762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773804.712729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773804.776741:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773804.776742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773804.776834:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773804.904748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773804.904748:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773804.904748:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773804.904762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773804.968775:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773804.968791:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773804.968791:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773804.968793:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773804.968824:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773804.968831:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773805.096772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773805.160736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773805.224710:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773805.288757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773805.288757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773805.288757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773805.288765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773805.352777:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773805.352787:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773805.352821:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773805.480748:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773805.480748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773805.544750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773805.608728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773805.672745:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773805.672745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773805.672745:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773805.736719:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773805.800751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773805.800751:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773805.800751:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773805.928785:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773805.928786:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773805.928786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773805.928786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773805.992747:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773805.992747:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773805.992752:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773805.992754:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773805.992774:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773805.992779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773806.120730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773806.184744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773806.248716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773806.312727:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773806.312734:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773806.312737:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773806.312783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773806.376747:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773806.376747:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773806.376795:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773806.504758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773806.504758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773806.568767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773806.632750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773806.696714:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773806.696729:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773806.696733:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773806.760722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773806.824742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773806.824742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773806.824743:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773806.952747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773806.952747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773806.952748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773806.952748:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773807.016712:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773807.016712:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773807.016713:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773807.016735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773807.016754:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773807.016758:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773807.144718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773807.208751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773807.272712:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773807.272718:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773807.272723:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773807.272727:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773807.272735:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2803 00000100:00080000:0.0:1790773807.272750:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773807.272761:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773807.272794:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534759168/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773807.272820:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773807.272822:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773807.272824:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773807.272827:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2803 00000100:00080000:0.0:1790773807.272830:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773807.272834:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773807.272841:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773807.272852:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534759296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773807.272860:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534759168:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773807.272861:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773807.272868:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773807.272876:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773807.272883:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2803 00000100:00080000:0.0:1790773807.272885:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773807.272888:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773807.272892:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773807.272901:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534759424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773807.272908:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773807.272910:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773807.272911:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773807.272915:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773807.272918:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2803 00000100:00080000:0.0:1790773807.272922:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773807.272926:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773807.272930:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773807.272938:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534759552/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773807.272944:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773807.272946:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773807.272947:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773807.272950:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773807/real 1790773807] req@ffff9c4ec7a7d180 x1877759534759168/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773807.272953:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2803 00000100:00080000:0.0:1790773807.272958:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773807.272963:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773807.272967:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773807.272971:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534759168/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773807.272977:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534759680/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773807.272981:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773807.272987:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773807.272988:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773807.272993:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1790773807.272996:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534759168:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773807.272997:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773807.273004:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2803 00000100:00080000:0.0:1790773807.273006:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773807.273009:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773807.273012:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773807.273018:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534759296:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773807.273021:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534759808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773807.273029:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773807.273032:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773807.273033:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773807.273038:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773807.273039:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2803 00000100:00080000:0.0:1790773807.273046:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773807.273052:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773807.273054:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773807/real 1790773807] req@ffff9c4ec7a7c700 x1877759534759296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773807.273058:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773807.273076:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534759296/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773807.273077:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534759936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773807.273108:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773807.273112:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773807.273114:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773807.273116:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773807.273118:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2803 00000100:00080000:0.0:1790773807.273124:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773807.273124:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773807.273133:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534759296:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773807.273134:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773807.273140:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773807.273149:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534760064/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773807.273158:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773807.273161:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534759424:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773807.273163:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773807.273167:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773807.273173:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2803 00000100:00080000:0.0:1790773807.273176:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773807.273188:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773807.273190:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773807.273197:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773807.273203:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773807/real 1790773807] req@ffff9c4ec7a7e300 x1877759534759424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773807.273212:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534760192/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773807.273220:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534759424/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773807.273233:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773807.273237:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773807.273241:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534759424:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773807.273249:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534759552:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773807.273268:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773807.273278:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773807/real 1790773807] req@ffff9c4ec7a7fb80 x1877759534759552/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773807.273300:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534759552/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773807.273310:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773807.273314:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773807.273317:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534759552:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773807.273323:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534759680:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773807.273364:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773807.273375:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773807/real 1790773807] req@ffff9c4ec7a7ca80 x1877759534759680/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773807.273387:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534759680/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773807.273396:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773807.273399:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773807.273403:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534759680:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773807.273418:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534759808:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773807.273437:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773807.273447:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773807/real 1790773807] req@ffff9c4ec7a7c380 x1877759534759808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773807.273466:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534759808/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773807.273492:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773807.273496:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773807.273499:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534759808:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773807.273505:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534759936:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773807.273523:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773807.273533:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773807/real 1790773807] req@ffff9c4ec7a7d880 x1877759534759936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773807.273545:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534759936/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773807.273554:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773807.273557:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773807.273560:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534759936:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773807.273566:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534760064:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773807.273583:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773807.273592:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773807/real 1790773807] req@ffff9c4ec7a7ed80 x1877759534760064/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773807.273603:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534760064/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773807.273619:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773807.273623:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773807.273626:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534760064:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773807.273632:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534760192:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773807.273648:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773807.273680:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773807/real 1790773807] req@ffff9c4ec7a7df80 x1877759534760192/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773807.273692:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534760192/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773807.273701:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773807.273704:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773807.273707:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534760192:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773807.336743:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773807.336743:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773807.336743:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773807.336746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773807.400747:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773807.400774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773807.400928:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773807.528720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773807.528723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773807.592741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773807.656729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773807.720748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773807.720748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773807.720749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773807.784700:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773807.848742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773807.848742:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773807.848742:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773807.976742:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773807.976742:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773807.976742:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773807.976742:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773808.040742:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773808.040742:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773808.040742:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773808.040742:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773808.040772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773808.040775:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773808.168760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773808.232740:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773808.296711:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773808.360758:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773808.360758:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773808.360759:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773808.360760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773808.424723:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773808.424753:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773808.424810:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773808.552747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773808.552747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773808.616749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773808.680729:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773808.744812:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773808.744813:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773808.744813:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773808.808758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773808.872806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773808.872807:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773808.872807:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773809.000783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773809.000783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773809.000786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773809.000787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773809.064748:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773809.064753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773809.064760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773809.064783:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773809.064800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773809.064808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773809.192760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773809.256780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773809.320716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773809.384774:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773809.384777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773809.384777:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773809.384777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773809.448760:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773809.448798:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773809.448813:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773809.576738:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773809.576738:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773809.640730:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773809.704764:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773809.768747:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773809.768755:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773809.768757:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773809.832754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773809.896746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773809.896753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773809.896756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773810.024749:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773810.024766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773810.024768:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773810.024768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773810.088742:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773810.088760:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773810.088761:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773810.088764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773810.088797:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773810.088806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773810.216746:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773810.280712:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773810.344726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773810.408795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773810.408795:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773810.408795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773810.408797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773810.472757:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773810.472822:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773810.472878:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773810.600766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773810.600767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773810.664757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773810.728736:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773810.792776:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773810.792776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773810.792778:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773810.856729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773810.920753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773810.920753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773810.920764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773811.048785:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773811.048785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773811.048785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773811.048816:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773811.112703:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773811.112732:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773811.112745:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773811.112753:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773811.112760:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773811.112774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773811.240718:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773811.304738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773811.368719:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773811.432713:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773811.432719:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773811.432723:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773811.432762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773811.496765:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773811.496791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773811.496827:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773811.624740:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773811.624742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773811.688750:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773811.752767:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773811.816740:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773811.816741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773811.816771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773811.880724:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773811.944762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773811.944771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773811.944771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773812.072769:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773812.072769:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773812.072781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773812.072781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773812.136738:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773812.136738:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773812.136749:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773812.136750:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773812.136784:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773812.136802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773812.264719:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773812.328751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773812.392725:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790773812.392732:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773812.392733:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773812.392737:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773812.392746:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2808 00000100:00080000:0.0:1790773812.392755:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773812.392765:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773812.392798:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534760320/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773812.392821:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773812.392824:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773812.392826:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773812.392829:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2808 00000100:00080000:0.0:1790773812.392831:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773812.392834:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773812.392842:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773812.392853:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534760448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773812.392860:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773812.392862:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773812.392864:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773812.392866:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2808 00000100:00080000:0.0:1790773812.392868:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773812.392871:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773812.392874:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773812.392880:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534760576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773812.392887:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773812.392890:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773812.392891:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773812.392894:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2808 00000100:00080000:0.0:1790773812.392896:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773812.392899:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773812.392903:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773812.392910:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534760704/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773812.392918:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534760320:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773812.392919:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773812.392927:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773812.392929:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773812.392932:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2808 00000100:00080000:0.0:1790773812.392934:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773812.392937:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773812.392940:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773812.392956:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534760832/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773812.392964:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773812.392968:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773812.392971:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773812.392975:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2808 00000800:00000100:1.0:1790773812.392978:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773812.392983:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773812.392992:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773812.392999:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773812.393015:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534760960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773812.393017:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773812/real 1790773812] req@ffff9c4ec7a7f480 x1877759534760320/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773812.393049:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773812.393052:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534760320/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773812.393056:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773812.393060:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773812.393066:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2808 00000100:00080000:0.0:1790773812.393074:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773812.393081:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773812.393084:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773812.393088:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773812.393093:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534760320:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773812.393094:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773812.393102:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534760448:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773812.393111:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534761088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773812.393120:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773812.393123:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773812.393125:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773812.393128:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773812.393133:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2808 00000100:00100000:1.0:1790773812.393137:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773812/real 1790773812] req@ffff9c4ec7a7df80 x1877759534760448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773812.393138:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773812.393148:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773812.393155:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773812.393160:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534760448/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773812.393172:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534761216/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773812.393175:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773812.393183:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773812.393189:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534760448:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773812.393194:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773812.393197:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534760576:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773812.393201:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773812.393205:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773812.393211:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2808 00000100:00080000:0.0:1790773812.393216:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773812.393219:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773812.393225:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773812.393232:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773812.393236:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773812/real 1790773812] req@ffff9c4ec7a7ed80 x1877759534760576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773812.393261:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534761344/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773812.393271:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534760576/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773812.393283:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773812.393288:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773812.393290:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534760576:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773812.393311:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534760704:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773812.393328:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773812.393336:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773812/real 1790773812] req@ffff9c4ec7a7d880 x1877759534760704/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773812.393344:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534760704/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773812.393351:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773812.393353:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773812.393356:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534760704:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773812.393360:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534760832:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773812.393370:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773812.393376:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773812/real 1790773812] req@ffff9c4ec7a7c380 x1877759534760832/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773812.393383:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534760832/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773812.393389:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773812.393391:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773812.393393:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534760832:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773812.393397:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534760960:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773812.393407:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773812.393412:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773812/real 1790773812] req@ffff9c4ec7a7ca80 x1877759534760960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773812.393420:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534760960/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773812.393435:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773812.393437:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773812.393439:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534760960:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773812.393443:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534761088:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773812.393453:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773812.393458:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773812/real 1790773812] req@ffff9c4ec7a7fb80 x1877759534761088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773812.393465:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534761088/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773812.393471:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773812.393473:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773812.393475:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534761088:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773812.393479:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534761216:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773812.393489:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773812.393494:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773812/real 1790773812] req@ffff9c4ec7a7e300 x1877759534761216/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773812.393502:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534761216/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773812.393509:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773812.393511:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773812.393513:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534761216:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773812.393517:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534761344:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773812.393527:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773812.393532:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773812/real 1790773812] req@ffff9c4ec7a7c700 x1877759534761344/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773812.393539:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534761344/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773812.393546:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773812.393548:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773812.393550:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534761344:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773812.456759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773812.456759:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773812.456759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773812.456770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773812.520737:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773812.520761:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773812.520799:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773812.648734:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773812.648744:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773812.712767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773812.776734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773812.840782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773812.840782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773812.840782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773812.904738:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773812.968761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773812.968779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773812.968779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773813.096814:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773813.096820:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773813.096823:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773813.096824:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773813.160768:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773813.160768:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773813.160775:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773813.160776:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773813.160809:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773813.160830:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773813.288767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773813.352766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773813.416709:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773813.480749:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773813.480749:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773813.480780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773813.480846:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773813.544761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773813.544808:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773813.544846:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773813.672738:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773813.672739:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773813.736778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773813.800750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773813.864732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773813.864733:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773813.864763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773813.928760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773813.992798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773813.992798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773813.992798:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773814.120764:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773814.120765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773814.120765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773814.120806:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773814.184760:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773814.184760:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773814.184760:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773814.184803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773814.184824:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773814.184824:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773814.312726:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773814.376814:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773814.440782:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773814.504768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773814.504794:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773814.504809:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773814.504810:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773814.568742:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773814.568786:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773814.568861:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773814.696754:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773814.696755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773814.760781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773814.824805:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773814.888804:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773814.888804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773814.888809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773814.952730:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773815.016746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773815.016750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773815.016779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773815.144813:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773815.144814:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773815.144814:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773815.144814:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773815.208790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773815.208791:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773815.208791:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773815.208797:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773815.208871:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773815.208873:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773815.336772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773815.400807:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773815.464713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773815.528752:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773815.528752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773815.528765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773815.528766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773815.592776:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773815.592797:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773815.592858:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773815.720758:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773815.720768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773815.784771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773815.848782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773815.912723:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773815.912723:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773815.912732:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773815.976770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773816.040759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773816.040759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773816.040759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773816.168755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773816.168755:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773816.168756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773816.168791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773816.232729:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773816.232751:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773816.232771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773816.232785:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773816.232869:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773816.232895:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773816.360728:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773816.424711:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773816.488724:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773816.552789:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773816.552789:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773816.552800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773816.552801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773816.616746:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773816.616833:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773816.616863:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773816.744745:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773816.744745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773816.808770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773816.872755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773816.936738:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773816.936738:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773816.936742:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773817.000731:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773817.064717:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773817.064724:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773817.064729:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773817.192752:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773817.192752:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773817.192753:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773817.192754:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773817.256750:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773817.256750:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773817.256752:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773817.256767:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773817.256793:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773817.256799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773817.384733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773817.448723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773817.512743:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773817.512747:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773817.512756:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773817.512761:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773817.512772:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2813 00000100:00080000:0.0:1790773817.512785:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773817.512797:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773817.512837:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534761472/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773817.512876:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773817.512880:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773817.512882:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773817.512887:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2813 00000100:00080000:0.0:1790773817.512891:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773817.512897:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773817.512907:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773817.512924:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534761600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773817.512933:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534761472:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773817.512950:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773817.512954:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773817.512957:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773817.512963:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2813 00000100:00080000:0.0:1790773817.512968:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773817.512991:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773817.513001:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773817.513007:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773817.513025:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534761728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773817.513037:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773817.513039:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773817/real 1790773817] req@ffff9c4ec7a7d180 x1877759534761472/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773817.513046:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773817.513051:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773817.513060:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2813 00000100:00080000:0.0:1790773817.513066:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773817.513066:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534761472/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773817.513078:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773817.513079:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773817.513087:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790773817.513089:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773817.513091:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534761472:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773817.513110:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534761856/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773817.513116:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534761600:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773817.513136:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773817.513157:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773817.513159:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773817.513165:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2813 00000100:00080000:0.0:1790773817.513169:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773817.513174:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773817.513180:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773817.513193:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534761984/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773817.513204:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773817.513208:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773817.513210:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773817.513214:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2813 00000100:00080000:0.0:1790773817.513218:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773817.513223:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773817.513229:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773817.513239:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534762112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773817.513250:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773817.513253:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773817.513256:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773817.513271:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2813 00000100:00080000:0.0:1790773817.513275:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773817.513287:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773817.513291:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773817.513297:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773817.513306:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773817/real 1790773817] req@ffff9c4ec7a7c700 x1877759534761600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773817.513316:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534762240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773817.513321:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534761600/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773817.513348:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773817.513353:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773817.513353:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773817.513358:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773817.513359:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773817.513366:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534761600:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773817.513370:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2813 00000100:00100000:1.0:1790773817.513375:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534761728:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773817.513378:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773817.513389:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773817.513395:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:0.0:1790773817.513396:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773817.513405:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773817/real 1790773817] req@ffff9c4ec7a7e300 x1877759534761728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773817.513414:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534762368/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773817.513416:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534761728/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773817.513445:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773817.513448:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773817.513450:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534761728:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773817.513459:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773817.513466:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773817.513466:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534761856:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773817.513472:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773817.513480:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2813 00000800:00000100:1.0:1790773817.513485:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773817.513487:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773817.513496:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773817/real 1790773817] req@ffff9c4ec7a7fb80 x1877759534761856/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773817.513497:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773817.513507:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773817.513521:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534761856/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773817.513530:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534762496/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773817.513530:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773817.513538:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773817.513545:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534761856:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773817.513555:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534761984:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773817.513568:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773817.513578:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773817/real 1790773817] req@ffff9c4ec7a7ca80 x1877759534761984/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773817.513587:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534761984/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773817.513593:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773817.513595:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773817.513598:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534761984:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773817.513601:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534762112:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773817.513610:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773817.513618:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773817/real 1790773817] req@ffff9c4ec7a7c380 x1877759534762112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773817.513631:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534762112/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773817.513680:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773817.513685:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773817.513689:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534762112:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773817.513696:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534762240:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773817.513717:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773817.513728:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773817/real 1790773817] req@ffff9c4ec7a7d880 x1877759534762240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773817.513740:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534762240/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773817.513748:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773817.513752:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773817.513755:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534762240:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773817.513761:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534762368:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773817.513780:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773817.513792:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773817/real 1790773817] req@ffff9c4ec7a7ed80 x1877759534762368/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773817.513805:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534762368/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773817.513816:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773817.513819:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773817.513823:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534762368:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773817.513842:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534762496:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773817.513864:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773817.513876:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773817/real 1790773817] req@ffff9c4ec7a7df80 x1877759534762496/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773817.513890:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534762496/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773817.513900:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773817.513904:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773817.513908:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534762496:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773817.576813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773817.576815:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773817.576815:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773817.576816:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773817.640812:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773817.640819:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773817.640913:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773817.768791:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773817.768905:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773817.832761:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773817.896791:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773817.960820:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773817.960821:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773817.960821:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773818.024765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773818.088792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773818.088792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773818.088797:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773818.216739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773818.216742:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773818.216757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773818.216760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773818.280768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773818.280770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773818.280794:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773818.280797:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773818.280812:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773818.280829:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773818.408779:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773818.472774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773818.536753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773818.600756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773818.600756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773818.600757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773818.600759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773818.664745:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773818.664815:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773818.664850:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773818.792725:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773818.792733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773818.856779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773818.920727:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773818.984754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773818.984754:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773818.984754:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773819.048732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773819.112757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773819.112757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773819.112757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773819.240773:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773819.240773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773819.240773:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773819.240778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773819.304734:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773819.304735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773819.304740:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773819.304762:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773819.304785:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773819.304797:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773819.432749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773819.496751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773819.560776:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773819.624742:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773819.624784:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773819.624784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773819.624784:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773819.688766:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773819.688800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773819.688864:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773819.816734:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773819.816734:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773819.880749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773819.944772:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773820.008736:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773820.008769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773820.008770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773820.072766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773820.136753:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773820.136753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773820.136753:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773820.264714:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773820.264718:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773820.264730:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773820.264761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773820.328748:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773820.328748:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773820.328748:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773820.328793:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773820.328804:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773820.328807:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773820.456701:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773820.520745:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773820.584705:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773820.648801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773820.648802:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773820.648803:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773820.648811:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00010000:00010000:1.0:1790773820.712801:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773820.712801:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773820.712816:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773820.840709:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773820.840756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773820.904746:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773820.968782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773821.032752:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773821.032752:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773821.032752:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773821.096729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773821.160764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773821.160764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773821.160764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773821.288732:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773821.288764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773821.288766:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773821.288768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773821.352767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773821.352771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773821.352772:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773821.352774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773821.352822:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773821.352822:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773821.480745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773821.544774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773821.608740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773821.672763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773821.672763:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773821.672796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773821.672796:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773821.736780:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773821.736792:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773821.736810:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773821.864707:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773821.864729:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773821.928777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773821.992785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773822.056762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773822.056762:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773822.056763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773822.120735:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773822.184764:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773822.184764:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773822.184764:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773822.312786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773822.312798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773822.312817:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773822.312826:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773822.376752:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773822.376791:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773822.376792:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773822.376792:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773822.376803:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773822.376821:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773822.504733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773822.568777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773822.632726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773822.632750:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773822.632759:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773822.632764:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773822.632774:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2818 00000100:00080000:2.0:1790773822.632786:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773822.632798:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773822.632844:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534762624/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773822.632868:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773822.632872:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773822.632873:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773822.632876:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2818 00000100:00080000:2.0:1790773822.632879:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773822.632882:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773822.632893:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773822.632905:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534762752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773822.632914:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773822.632919:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534762624:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773822.632919:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773822.632926:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773822.632937:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2818 00000100:00080000:2.0:1790773822.632940:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773822.632944:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773822.632947:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773822.632958:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534762880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773822.632966:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773822.632969:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773822.632970:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773822.632975:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2818 00000100:00080000:2.0:1790773822.632979:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773822.632982:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773822.632988:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:1.0:1790773822.632996:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773822.632998:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534763008/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773822.633008:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773822.633010:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773822.633013:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773822.633018:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2818 00000100:00080000:2.0:1790773822.633022:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773822.633026:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773822.633029:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773822.633062:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534763136/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773822.633071:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773822/real 1790773822] req@ffff9c4efbcc0380 x1877759534762624/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773822.633079:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773822.633092:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773822.633097:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773822.633106:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2818 00000100:00080000:2.0:1790773822.633110:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773822.633112:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534762624/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773822.633119:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773822.633133:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773822.633138:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773822.633143:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773822.633147:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534762624:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773822.633152:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534763264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773822.633163:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773822.633166:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773822.633168:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773822.633177:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2818 00000100:00100000:1.0:1790773822.633178:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534762752:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773822.633189:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773822.633195:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773822.633200:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773822.633210:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534763392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773822.633215:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773822.633230:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773822/real 1790773822] req@ffff9c4efbcc2300 x1877759534762752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773822.633234:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773822.633246:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534762752/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773822.633246:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773822.633253:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773822.633269:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2818 00000100:00080000:2.0:1790773822.633276:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773822.633282:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773822.633288:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773822.633292:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773822.633299:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534762752:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773822.633304:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534762880:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790773822.633307:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:1.0:1790773822.633320:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773822.633329:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534763520/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773822.633334:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773822/real 1790773822] req@ffff9c4efbcc1f80 x1877759534762880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773822.633368:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773822.633372:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534762880/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773822.633379:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773822.633392:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773822.633393:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773822.633408:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773822.633414:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2818 00000100:00100000:1.0:1790773822.633419:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534762880:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773822.633425:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773822.633433:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534763008:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773822.633438:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773822.633447:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:1.0:1790773822.633458:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773822.633463:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534763648/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773822.633473:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773822/real 1790773822] req@ffff9c4efbcc1c00 x1877759534763008/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773822.633493:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534763008/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773822.633506:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773822.633510:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773822.633514:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534763008:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773822.633522:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534763136:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773822.633544:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773822.633556:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773822/real 1790773822] req@ffff9c4efbcc3100 x1877759534763136/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773822.633564:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534763136/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773822.633571:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773822.633574:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773822.633576:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534763136:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773822.633581:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534763264:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773822.633594:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773822.633600:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773822/real 1790773822] req@ffff9c4efbcc2680 x1877759534763264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773822.633609:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534763264/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773822.633620:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773822.633623:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773822.633625:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534763264:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773822.633642:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534763392:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773822.633679:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773822.633687:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773822/real 1790773822] req@ffff9c4efbcc1880 x1877759534763392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773822.633695:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534763392/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773822.633702:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773822.633704:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773822.633707:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534763392:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773822.633712:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534763520:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773822.633723:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773822.633728:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773822/real 1790773822] req@ffff9c4efbcc1500 x1877759534763520/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773822.633737:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534763520/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773822.633748:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773822.633751:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773822.633753:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534763520:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773822.633757:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534763648:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773822.633767:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773822.633773:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773822/real 1790773822] req@ffff9c4efbcc0700 x1877759534763648/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773822.633780:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534763648/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773822.633786:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773822.633788:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773822.633790:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534763648:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773822.696800:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773822.696801:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773822.696801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773822.696809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773822.760765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773822.760776:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773822.760870:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773822.888735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773822.888743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773822.952818:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773823.016766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773823.080781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773823.080783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773823.080814:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773823.144747:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773823.208757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773823.208775:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773823.208787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773823.336735:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773823.336735:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773823.336742:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773823.336759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773823.400731:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773823.400734:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773823.400735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773823.400735:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773823.400764:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773823.400766:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773823.528753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773823.592775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773823.656764:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773823.720755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773823.720756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773823.720756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773823.720756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773823.784752:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773823.784752:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773823.784835:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773823.912754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773823.912754:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773823.976794:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773824.040750:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773824.104729:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773824.104733:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773824.104735:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773824.168723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773824.232758:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773824.232758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773824.232758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773824.360810:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773824.360810:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773824.360812:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773824.360815:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773824.424741:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773824.424799:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773824.424799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773824.424800:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773824.424831:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773824.424855:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773824.552775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773824.616812:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773824.680789:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773824.744752:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773824.744752:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773824.744752:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773824.744793:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773824.808797:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773824.808803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773824.808853:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773824.936773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773824.936773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773825.000826:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773825.064787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773825.128744:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773825.128779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773825.128779:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773825.192722:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773825.256763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773825.256772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773825.256785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773825.384728:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773825.384733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773825.384771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773825.384788:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773825.448755:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773825.448774:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773825.448779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773825.448803:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773825.448820:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773825.448833:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773825.576785:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773825.640780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773825.704716:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773825.768806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773825.768806:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773825.768824:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773825.768828:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773825.832732:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773825.832781:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773825.832784:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773825.960795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773825.960798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773826.024786:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773826.088768:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773826.152722:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773826.152726:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773826.152749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773826.216760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773826.280745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773826.280745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773826.280745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773826.408741:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773826.408741:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773826.408746:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773826.408747:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773826.472787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773826.472788:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773826.472788:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773826.472788:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773826.472849:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773826.472850:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773826.600774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773826.664751:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773826.728737:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773826.792839:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773826.792839:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773826.792839:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773826.792839:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773826.856774:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773826.856774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773826.856897:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773826.984729:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773826.984729:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773827.048754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773827.112809:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773827.176753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773827.176753:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773827.176763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773827.240781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773827.304759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773827.304761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773827.304781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773827.432756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773827.432778:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773827.432778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773827.432888:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773827.496748:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773827.496750:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773827.496789:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773827.496810:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773827.496816:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773827.496830:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773827.624762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773827.688765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773827.752727:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773827.752784:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773827.752795:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773827.752801:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773827.752812:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2823 00000100:00080000:2.0:1790773827.752824:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773827.752836:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773827.752875:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534763776/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773827.752906:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773827.752911:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773827.752914:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773827.752919:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2823 00000100:00080000:2.0:1790773827.752923:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773827.752929:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773827.752954:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773827.752972:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534763904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773827.752985:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773827.752989:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773827.752992:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773827.752996:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534763776:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773827.753001:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2823 00000100:00080000:2.0:1790773827.753007:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773827.753013:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773827.753020:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773827.753038:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534764032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773827.753050:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773827.753055:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1790773827.753059:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773827.753060:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773827.753085:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2823 00000100:00080000:2.0:1790773827.753091:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773827.753098:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773827.753107:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773827.753111:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773827/real 1790773827] req@ffff9c4efbcc0a80 x1877759534763776/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773827.753124:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534764160/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773827.753131:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534763776/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773827.753144:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773827.753147:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773827.753149:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773827.753155:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773827.753156:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773827.753164:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2823 00000100:00100000:1.0:1790773827.753167:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534763776:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773827.753171:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773827.753178:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534763904:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773827.753181:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773827.753190:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773827.753206:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534764288/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773827.753215:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773827.753222:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773827.753227:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773827.753230:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773827.753234:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773827/real 1790773827] req@ffff9c4efbcc2a00 x1877759534763904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773827.753239:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2823 00000100:00080000:2.0:1790773827.753247:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773827.753255:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773827.753260:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534763904/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773827.753263:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773827.753281:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534764416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773827.753284:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773827.753292:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773827.753299:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534763904:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773827.753304:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773827.753307:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773827.753309:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773827.753319:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2823 00000100:00080000:2.0:1790773827.753324:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773827.753330:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773827.753336:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773827.753341:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534764032:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773827.753362:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534764544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773827.753363:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773827.753374:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773827/real 1790773827] req@ffff9c4efbcc0700 x1877759534764032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773827.753380:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773827.753390:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773827.753392:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773827.753397:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534764032/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773827.753402:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2823 00000100:00080000:2.0:1790773827.753409:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773827.753418:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773827.753420:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773827.753426:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773827.753430:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773827.753433:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534764032:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773827.753444:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534764160:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773827.753449:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534764672/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773827.753464:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773827.753465:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773827.753472:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773827.753475:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773827.753478:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773827/real 1790773827] req@ffff9c4efbcc1500 x1877759534764160/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773827.753484:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2823 00000100:00100000:1.0:1790773827.753492:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534764160/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773827.753493:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773827.753505:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773827.753505:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773827.753514:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773827.753516:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773827.753517:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534764160:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773827.753526:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534764288:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773827.753537:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534764800/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773827.753544:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773827.753554:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773827/real 1790773827] req@ffff9c4efbcc1880 x1877759534764288/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773827.753564:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534764288/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773827.753570:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773827.753573:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773827.753576:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534764288:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773827.753580:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534764416:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773827.753590:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773827.753596:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773827/real 1790773827] req@ffff9c4efbcc2680 x1877759534764416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773827.753604:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534764416/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773827.753615:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773827.753616:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773827.753619:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534764416:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773827.753630:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534764544:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773827.753641:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773827.753647:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773827/real 1790773827] req@ffff9c4efbcc3100 x1877759534764544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773827.753680:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534764544/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773827.753686:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773827.753688:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773827.753690:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534764544:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773827.753694:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534764672:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773827.753707:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773827.753714:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773827/real 1790773827] req@ffff9c4efbcc1c00 x1877759534764672/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773827.753721:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534764672/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773827.753733:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773827.753736:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773827.753738:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534764672:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773827.753742:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534764800:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773827.753752:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773827.753757:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773827/real 1790773827] req@ffff9c4efbcc1f80 x1877759534764800/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773827.753764:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534764800/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773827.753771:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773827.753773:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773827.753775:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534764800:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773827.816732:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773827.816783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773827.816784:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773827.816788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773827.880773:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773827.880775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773827.880906:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773828.008768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773828.008771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773828.072749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773828.136766:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773828.200701:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773828.200724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773828.200724:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773828.264715:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773828.328771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773828.328771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773828.328872:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773828.456733:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773828.456736:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773828.456736:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773828.456737:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773828.520704:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773828.520733:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773828.520738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773828.520761:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773828.520761:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773828.520775:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773828.648717:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773828.712790:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773828.776731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773828.840786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773828.840797:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773828.840802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773828.840813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773828.904772:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773828.904772:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773828.904914:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773829.032767:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773829.032769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773829.096814:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773829.160770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773829.224797:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773829.224797:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773829.224806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773829.288764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773829.352803:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773829.352804:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773829.352804:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773829.480781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773829.480781:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773829.480781:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773829.480782:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773829.544741:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773829.544774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773829.544775:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773829.544776:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773829.544795:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773829.544820:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773829.672763:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773829.736775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773829.800794:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773829.864808:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773829.864808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773829.864832:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773829.864832:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773829.928760:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773829.928773:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773829.928847:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773830.056756:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773830.056762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773830.120743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773830.184732:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773830.248763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773830.248764:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773830.248764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773830.312799:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773830.376769:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773830.376769:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773830.376769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773830.504759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773830.504759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773830.504759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773830.504794:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773830.568723:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773830.568755:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773830.568777:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773830.568778:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773830.568780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773830.568793:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773830.696760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773830.760804:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773830.824751:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773830.888773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773830.888773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773830.888774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773830.888784:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773830.952807:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773830.952822:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773830.952926:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773831.080748:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773831.080748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773831.144753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773831.208836:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773831.272739:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773831.272747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773831.272747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773831.336819:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773831.400811:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773831.400811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773831.400812:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773831.528755:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773831.528756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773831.528761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773831.528789:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773831.592786:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773831.592797:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773831.592797:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773831.592797:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773831.592825:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773831.592891:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773831.720741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773831.784732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773831.848749:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773831.912792:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773831.912792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773831.912793:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773831.912800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773831.976786:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773831.976790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773831.976840:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773832.104759:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773832.104778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773832.168770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773832.232805:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773832.296820:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773832.296821:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773832.296821:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773832.360789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773832.424750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773832.424762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773832.424763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773832.552795:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773832.552795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773832.552795:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773832.552796:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773832.616821:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773832.616823:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773832.616823:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773832.616830:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773832.616887:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773832.616900:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773832.744788:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773832.808796:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773832.872750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773832.872760:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773832.872773:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773832.872779:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773832.872793:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2828 00000100:00080000:2.0:1790773832.872808:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773832.872824:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773832.872893:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534764928/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773832.872932:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773832.872937:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773832.872939:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773832.872945:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2828 00000100:00080000:2.0:1790773832.872949:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773832.872958:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773832.872973:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773832.872994:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534764928:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773832.872995:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534765056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773832.873022:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773832.873027:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773832.873030:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773832.873037:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2828 00000100:00080000:2.0:1790773832.873043:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773832.873051:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773832.873061:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773832.873069:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773832.873083:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534765184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773832.873107:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773832.873109:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773832/real 1790773832] req@ffff9c4efbcc2300 x1877759534764928/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773832.873126:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773832.873133:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773832.873145:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2828 00000100:00100000:1.0:1790773832.873146:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534764928/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773832.873158:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773832.873162:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773832.873168:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773832.873173:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773832.873178:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534764928:192.168.204.150@tcp:250:kworker.0 00000008:00100000:2.0:1790773832.873187:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773832.873204:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534765056:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773832.873210:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534765312/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773832.873227:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773832.873228:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773832.873235:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773832.873238:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773832.873240:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773832/real 1790773832] req@ffff9c4efbcc0380 x1877759534765056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773832.873251:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2828 00000100:00100000:1.0:1790773832.873259:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534765056/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773832.873263:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773832.873280:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773832.873283:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773832.873289:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773832.873289:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773832.873295:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534765056:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773832.873301:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534765184:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773832.873320:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773832.873326:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534765440/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773832.873335:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773832/real 1790773832] req@ffff9c4efbcc1f80 x1877759534765184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773832.873370:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534765184/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773832.873372:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773832.873390:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773832.873393:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773832.873396:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773832.873400:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773832.873411:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2828 00000100:00100000:1.0:1790773832.873412:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534765184:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773832.873427:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773832.873436:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534765312:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773832.873439:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773832.873459:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790773832.873460:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773832.873469:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773832/real 1790773832] req@ffff9c4efbcc1c00 x1877759534765312/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773832.873483:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534765312/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773832.873491:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534765568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773832.873495:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773832.873504:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773832.873517:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534765312:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773832.873535:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534765440:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773832.873540:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773832.873546:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:1.0:1790773832.873550:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773832.873551:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773832.873566:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2828 00000100:00100000:1.0:1790773832.873568:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773832/real 1790773832] req@ffff9c4efbcc3100 x1877759534765440/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773832.873576:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773832.873583:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534765440/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773832.873591:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773832.873600:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773832.873604:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773832.873608:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773832.873612:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534765440:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773832.873624:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534765696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773832.873630:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534765568:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773832.873691:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773832.873695:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773832.873698:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773832.873706:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773832.873716:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2828 00000100:00080000:2.0:1790773832.873725:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773832.873727:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773832/real 1790773832] req@ffff9c4efbcc2680 x1877759534765568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773832.873737:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773832.873742:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534765568/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773832.873748:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773832.873757:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773832.873762:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773832.873766:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534765568:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773832.873771:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534765824/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773832.873781:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534765696:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773832.873789:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773832.873796:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773832.873802:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773832.873810:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2828 00000100:00080000:2.0:1790773832.873818:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773832.873826:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773832.873832:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:1.0:1790773832.873836:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773832.873872:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773832/real 1790773832] req@ffff9c4efbcc1880 x1877759534765696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773832.873872:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534765952/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773832.873957:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534765696/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773832.873966:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773832.873969:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773832.873972:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534765696:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773832.873987:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534765824:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773832.874004:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773832.874011:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773832/real 1790773832] req@ffff9c4efbcc1500 x1877759534765824/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773832.874019:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534765824/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773832.874026:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773832.874028:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773832.874031:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534765824:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773832.874035:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534765952:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773832.874047:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773832.874053:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773832/real 1790773832] req@ffff9c4efbcc0700 x1877759534765952/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773832.874060:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534765952/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773832.874066:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773832.874069:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773832.874071:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534765952:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773832.936757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773832.936761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773832.936787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773832.936788:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773833.000880:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773833.000905:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773833.000938:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773833.128776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773833.128788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773833.192785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773833.256823:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773833.320786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773833.320788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773833.320799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773833.384746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773833.448800:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773833.448801:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773833.448806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773833.576797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773833.576798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773833.576801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773833.576801:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773833.640782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773833.640782:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773833.640788:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773833.640795:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773833.640854:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773833.640856:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773833.768762:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773833.832731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773833.896740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773833.960800:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773833.960832:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773833.960838:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773833.960846:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773834.024751:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773834.024781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773834.024796:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773834.152755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773834.152755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773834.216754:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773834.280793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773834.344748:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773834.344748:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773834.344748:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773834.408736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773834.472748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773834.472748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773834.472748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773834.600756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773834.600756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773834.600757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773834.600781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773834.664816:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773834.664823:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773834.664828:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773834.664829:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773834.664847:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773834.664858:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773834.792780:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773834.856827:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773834.920726:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773834.984739:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773834.984739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773834.984764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773834.984778:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773835.048753:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773835.048759:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773835.048811:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773835.176745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773835.176746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773835.240740:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773835.304765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773835.368760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773835.368760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773835.368760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773835.432751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773835.496748:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773835.496748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773835.496871:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773835.624732:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773835.624733:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773835.624733:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773835.624783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773835.688744:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773835.688747:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773835.688760:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773835.688772:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773835.688812:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773835.688813:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773835.816745:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773835.880737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773835.944723:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773836.008816:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773836.008816:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773836.008825:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773836.008827:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773836.072761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773836.072772:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773836.072791:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773836.200721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773836.200721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773836.264756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773836.328753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773836.392795:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773836.392810:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773836.392811:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773836.456726:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773836.520733:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773836.520758:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773836.520758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773836.648831:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773836.648832:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773836.648837:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773836.648837:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773836.712739:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773836.712739:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773836.712781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773836.712781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773836.712818:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773836.712836:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773836.840736:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773836.904781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773836.968735:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773837.032800:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773837.032801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773837.032809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773837.032813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773837.096812:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773837.096815:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773837.096893:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773837.224740:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773837.224744:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773837.288724:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773837.352813:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773837.416769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773837.416770:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773837.416770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773837.480746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773837.544760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773837.544760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773837.544761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773837.672701:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773837.672734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773837.672734:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773837.672764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773837.736764:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773837.736787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773837.736788:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773837.736824:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773837.736825:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773837.736847:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773837.864774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773837.928767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773837.992755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773837.992759:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773837.992764:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773837.992768:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773837.992779:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2833 00000100:00080000:2.0:1790773837.992789:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773837.992800:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773837.992836:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534766080/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773837.992860:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773837.992863:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773837.992865:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773837.992869:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2833 00000100:00080000:2.0:1790773837.992872:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773837.992878:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773837.992888:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773837.992891:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534766080:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773837.992907:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534766208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773837.992921:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773837.992924:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773837.992926:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773837.992930:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2833 00000100:00080000:2.0:1790773837.992935:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773837.992940:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773837.992943:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773837.992952:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773837.992966:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534766336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773837.992977:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773837.992980:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773837.992984:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773837.992985:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773837/real 1790773837] req@ffff9c4efbcc2a00 x1877759534766080/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773837.992992:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2833 00000100:00080000:2.0:1790773837.993000:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773837.993010:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773837.993011:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534766080/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773837.993016:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00080000:1.0:1790773837.993020:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773837.993025:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773837.993029:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534766464/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773837.993030:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534766080:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773837.993043:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773837.993047:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773837.993050:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773837.993056:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2833 00000100:00080000:2.0:1790773837.993060:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773837.993061:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534766208:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773837.993067:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773837.993074:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773837.993087:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534766592/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773837.993094:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773837.993102:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773837.993105:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773837.993107:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773837.993112:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773837/real 1790773837] req@ffff9c4efbcc0a80 x1877759534766208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773837.993112:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2833 00000100:00080000:2.0:1790773837.993118:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773837.993125:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773837.993129:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773837.993130:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534766208/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773837.993141:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534766720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773837.993146:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773837.993148:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773837.993151:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773837.993152:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773837.993156:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773837.993158:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534766208:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773837.993162:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2833 00000100:00080000:2.0:1790773837.993166:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773837.993167:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534766336:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773837.993178:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773837.993183:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773837.993187:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773837.993196:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534766848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773837.993198:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773837/real 1790773837] req@ffff9c4efbcc0700 x1877759534766336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773837.993214:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773837.993218:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534766336/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773837.993222:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773837.993225:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773837.993227:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773837.993231:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2833 00000100:00080000:1.0:1790773837.993234:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773837.993235:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773837.993238:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534766336:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773837.993242:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773837.993246:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534766464:192.168.204.150@tcp:8:kworker.0 00000008:00100000:2.0:1790773837.993247:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773837.993260:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534766976/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773837.993267:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773837.993270:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773837.993272:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773837.993274:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773837.993278:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773837/real 1790773837] req@ffff9c4efbcc1500 x1877759534766464/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773837.993280:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2833 00000100:00080000:2.0:1790773837.993286:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773837.993291:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773837.993295:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534766464/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773837.993297:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773837.993307:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1790773837.993309:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534767104/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773837.993320:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773837.993324:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534766464:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773837.993338:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534766592:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773837.993349:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773837.993355:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773837/real 1790773837] req@ffff9c4efbcc1880 x1877759534766592/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773837.993363:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534766592/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773837.993369:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773837.993371:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773837.993373:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534766592:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773837.993376:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534766720:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773837.993385:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773837.993390:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773837/real 1790773837] req@ffff9c4efbcc2680 x1877759534766720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773837.993401:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534766720/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773837.993411:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773837.993413:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773837.993414:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534766720:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773837.993418:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534766848:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773837.993427:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773837.993432:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773837/real 1790773837] req@ffff9c4efbcc3100 x1877759534766848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773837.993438:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534766848/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773837.993443:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773837.993445:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773837.993446:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534766848:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773837.993450:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534766976:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773837.993458:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773837.993463:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773837/real 1790773837] req@ffff9c4efbcc1c00 x1877759534766976/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773837.993469:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534766976/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773837.993479:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773837.993482:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773837.993483:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534766976:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773837.993486:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534767104:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773837.993494:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773837.993499:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773837/real 1790773837] req@ffff9c4efbcc1f80 x1877759534767104/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773837.993506:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534767104/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773837.993511:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773837.993513:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773837.993520:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534767104:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773838.056756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773838.056756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773838.056756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773838.056766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773838.120733:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773838.120802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773838.120818:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773838.248751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773838.248751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773838.312759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773838.376805:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773838.440724:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773838.440724:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773838.440726:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773838.504814:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773838.568762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773838.568762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773838.568762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773838.696779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773838.696803:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773838.696810:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773838.696825:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773838.760758:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773838.760768:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773838.760781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773838.760790:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773838.760793:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773838.760800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773838.888707:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773838.952742:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773839.016713:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773839.080744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773839.080744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773839.080786:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773839.080807:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773839.144794:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773839.144811:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773839.144814:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773839.272736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773839.272736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773839.336734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773839.400814:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773839.464780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773839.464780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773839.464793:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773839.528776:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773839.592745:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773839.592766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773839.592767:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773839.720728:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773839.720801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773839.720802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773839.720802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773839.784772:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773839.784773:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773839.784777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773839.784788:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773839.784820:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773839.784822:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773839.912780:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773839.976765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773840.040745:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773840.104779:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773840.104779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773840.104788:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773840.104799:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773840.168762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773840.168827:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773840.168890:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773840.296780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773840.296780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773840.360797:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773840.424738:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773840.488772:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773840.488772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773840.488781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773840.552760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773840.616780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773840.616780:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773840.616801:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773840.744816:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773840.744820:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773840.744832:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773840.744860:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773840.808757:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773840.808765:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773840.808787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773840.808795:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773840.808811:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773840.808822:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773840.936844:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773841.000723:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773841.064753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773841.128774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773841.128775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773841.128787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773841.128787:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773841.192782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773841.192861:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773841.192868:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773841.320746:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773841.320746:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773841.384729:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773841.448735:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773841.512813:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773841.512816:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773841.512817:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773841.576800:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773841.640739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773841.640740:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773841.640777:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773841.768774:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773841.768774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773841.768798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773841.768803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773841.832728:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773841.832744:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773841.832753:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773841.832781:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773841.832799:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773841.832801:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773841.960785:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773842.024732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773842.088734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773842.152755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773842.152755:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773842.152756:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773842.152789:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773842.216772:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773842.216792:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773842.216933:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773842.344782:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773842.344784:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773842.408803:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773842.472769:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773842.536768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773842.536774:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773842.536774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773842.600815:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773842.664811:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773842.664813:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773842.664815:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773842.792773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773842.792814:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773842.792823:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773842.792823:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773842.856771:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773842.856774:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773842.856774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773842.856774:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773842.856824:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773842.856832:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773842.984831:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773843.048794:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773843.112757:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773843.112766:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:3.0:1790773843.112767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773843.112770:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773843.112788:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2839 00000100:00080000:2.0:1790773843.112798:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773843.112808:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773843.112852:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534767232/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773843.112890:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773843.112894:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773843.112896:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773843.112901:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2839 00000100:00080000:2.0:1790773843.112904:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773843.112909:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773843.112918:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773843.112931:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534767360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773843.112942:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773843.112945:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773843.112946:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773843.112955:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2839 00000100:00100000:1.0:1790773843.112959:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534767232:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773843.112960:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773843.112972:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773843.112978:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773843.112993:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534767488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773843.113004:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773843.113009:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773843.113012:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773843.113017:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2839 00000100:00080000:2.0:1790773843.113022:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773843.113030:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773843.113034:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:1.0:1790773843.113044:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773843.113050:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534767616/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773843.113069:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773843.113073:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773843.113075:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773843.113079:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2839 00000100:00080000:2.0:1790773843.113082:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773843.113088:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773843.113095:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773843.113100:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773843/real 1790773843] req@ffff9c4efbcc0380 x1877759534767232/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773843.113108:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534767744/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773843.113147:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773843.113152:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534767232/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773843.113154:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773843.113158:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773843.113169:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2839 00000100:00080000:2.0:1790773843.113178:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773843.113179:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773843.113186:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773843.113193:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773843.113197:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773843.113202:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534767232:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773843.113212:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534767872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773843.113222:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773843.113225:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773843.113227:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773843.113232:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2839 00000100:00100000:1.0:1790773843.113235:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534767360:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773843.113239:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773843.113245:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773843.113249:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773843.113261:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534768000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773843.113270:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773843.113273:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773843.113278:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773843.113280:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773843.113288:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2839 00000100:00100000:1.0:1790773843.113289:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773843/real 1790773843] req@ffff9c4efbcc2300 x1877759534767360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773843.113298:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773843.113309:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773843.113315:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773843.113319:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534767360/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773843.113327:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534768128/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773843.113363:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773843.113366:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773843.113369:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773843.113375:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2839 00000100:00080000:2.0:1790773843.113380:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773843.113381:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773843.113386:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773843.113392:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773843.113393:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773843.113402:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534768256/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773843.113403:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534767360:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773843.113426:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534767488:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773843.113456:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773843.113465:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773843/real 1790773843] req@ffff9c4efbcc1f80 x1877759534767488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773843.113475:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534767488/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773843.113482:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773843.113485:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773843.113487:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534767488:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773843.113493:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534767616:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773843.113512:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773843.113523:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773843/real 1790773843] req@ffff9c4efbcc1c00 x1877759534767616/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773843.113537:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534767616/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773843.113549:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773843.113553:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773843.113566:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534767616:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773843.113575:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534767744:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773843.113593:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773843.113602:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773843/real 1790773843] req@ffff9c4efbcc3100 x1877759534767744/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773843.113610:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534767744/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773843.113617:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773843.113620:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773843.113622:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534767744:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773843.113627:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534767872:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773843.113682:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773843.113694:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773843/real 1790773843] req@ffff9c4efbcc2680 x1877759534767872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773843.113704:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534767872/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773843.113719:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773843.113722:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773843.113726:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534767872:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773843.113753:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534768000:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773843.113777:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773843.113788:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773843/real 1790773843] req@ffff9c4efbcc1880 x1877759534768000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773843.113800:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534768000/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773843.113809:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773843.113813:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773843.113817:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534768000:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773843.113825:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534768128:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773843.113847:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773843.113857:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773843/real 1790773843] req@ffff9c4efbcc1500 x1877759534768128/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773843.113865:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534768128/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773843.113878:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773843.113881:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773843.113883:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534768128:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773843.113888:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534768256:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773843.113901:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773843.113907:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773843/real 1790773843] req@ffff9c4efbcc0700 x1877759534768256/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773843.113915:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534768256/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773843.113922:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773843.113924:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773843.113926:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534768256:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773843.176756:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773843.176756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773843.176786:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773843.176787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773843.240768:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773843.240803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773843.240878:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773843.368759:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773843.368759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773843.432779:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773843.496770:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773843.560807:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773843.560807:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773843.560818:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773843.624742:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773843.688721:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773843.688721:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773843.688725:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773843.816759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773843.816759:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773843.816760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773843.816760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773843.880757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773843.880757:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773843.880757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773843.880882:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773843.880898:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773843.880954:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773844.008760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773844.072706:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773844.136759:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773844.200747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773844.200747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773844.200747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773844.200747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773844.264749:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773844.264749:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773844.264801:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773844.392733:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773844.392733:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773844.456721:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773844.520745:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773844.584741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773844.584741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773844.584742:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773844.648734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773844.712728:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773844.712728:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773844.712745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773844.840773:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773844.840773:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773844.840773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773844.840887:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773844.904756:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773844.904756:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773844.904758:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773844.904921:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773844.904923:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773844.904963:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773845.032759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773845.096731:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773845.160763:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773845.224729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773845.224729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773845.224730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773845.224730:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773845.288748:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773845.288782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773845.288789:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773845.416723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773845.416724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773845.480728:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773845.544774:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773845.608750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773845.608750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773845.608750:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773845.672800:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773845.736762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773845.736763:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773845.736805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773845.864808:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773845.864808:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773845.864813:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773845.864825:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773845.928779:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773845.928781:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773845.928795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773845.928822:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773845.928825:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773845.928826:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773846.056767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773846.120801:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773846.184775:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773846.248763:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773846.248770:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773846.248777:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773846.248795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773846.312804:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773846.312811:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773846.312873:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773846.440748:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773846.440748:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773846.504753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773846.568759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773846.632745:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773846.632745:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773846.632745:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773846.696805:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773846.760773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773846.760779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773846.760799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773846.888701:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773846.888728:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773846.888735:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773846.888757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773846.952744:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773846.952744:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773846.952771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773846.952772:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773846.952803:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773846.952809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773847.080751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773847.144766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773847.208750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773847.272719:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773847.272724:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773847.272736:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773847.272755:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773847.336745:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773847.336745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773847.336801:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773847.464741:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773847.464742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773847.528734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773847.592739:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773847.656756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773847.656756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773847.656756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773847.720852:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773847.784746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773847.784746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773847.784751:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773847.912760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773847.912761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773847.912761:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773847.912855:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773847.976715:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773847.976715:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773847.976719:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773847.976738:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773847.976755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773847.976757:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773848.104748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773848.168752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773848.232752:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1790773848.232752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773848.232761:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773848.232766:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773848.232775:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2844 00000100:00080000:2.0:1790773848.232784:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773848.232795:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773848.232827:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534768384/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773848.232853:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773848.232856:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773848.232859:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773848.232863:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2844 00000100:00080000:2.0:1790773848.232866:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773848.232870:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773848.232879:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773848.232892:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534768512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773848.232904:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773848.232907:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534768384:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773848.232908:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773848.232911:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773848.232919:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2844 00000100:00080000:2.0:1790773848.232922:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773848.232927:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773848.232931:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773848.232943:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534768640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773848.232953:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773848.232957:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773848.232960:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773848.232964:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2844 00000100:00080000:2.0:1790773848.232968:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773848.232972:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773848.232975:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773848.232982:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773848.232993:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534768768/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773848.233003:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773848.233006:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773848.233008:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773848.233012:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2844 00000100:00080000:2.0:1790773848.233017:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773848.233018:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773848/real 1790773848] req@ffff9c4efbcc0a80 x1877759534768384/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773848.233025:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773848.233033:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773848.233045:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534768384/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773848.233047:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534768896/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773848.233074:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773848.233074:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773848.233081:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773848.233084:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773848.233084:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773848.233092:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2844 00000100:00100000:1.0:1790773848.233094:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534768384:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773848.233096:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773848.233104:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773848.233109:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773848.233119:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534769024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773848.233123:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534768512:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773848.233138:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773848.233142:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773848.233144:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773848.233149:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2844 00000800:00000100:1.0:1790773848.233154:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773848.233155:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773848.233162:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773848.233167:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773848.233173:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773848/real 1790773848] req@ffff9c4efbcc2a00 x1877759534768512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773848.233180:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534769152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773848.233194:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534768512/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773848.233196:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773848.233203:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773848.233207:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773848.233215:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2844 00000100:00080000:2.0:1790773848.233219:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773848.233225:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773848.233225:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773848.233233:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773848.233234:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773848.233241:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534768512:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773848.233245:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534769280/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773848.233252:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534768640:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773848.233258:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773848.233262:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773848.233265:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773848.233270:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2844 00000100:00080000:2.0:1790773848.233274:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773848.233279:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773848.233282:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773848.233288:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773848.233294:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773848/real 1790773848] req@ffff9c4efbcc0700 x1877759534768640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773848.233300:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534769408/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773848.233315:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534768640/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773848.233326:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773848.233330:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773848.233334:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534768640:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773848.233340:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534768768:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773848.233360:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773848.233370:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773848/real 1790773848] req@ffff9c4efbcc1500 x1877759534768768/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773848.233381:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534768768/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773848.233390:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773848.233393:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773848.233397:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534768768:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773848.233402:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534768896:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773848.233417:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773848.233425:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773848/real 1790773848] req@ffff9c4efbcc1880 x1877759534768896/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773848.233436:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534768896/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773848.233444:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773848.233447:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773848.233451:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534768896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773848.233469:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534769024:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773848.233487:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773848.233498:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773848/real 1790773848] req@ffff9c4efbcc2680 x1877759534769024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773848.233515:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534769024/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773848.233531:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773848.233534:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773848.233538:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534769024:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773848.233563:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534769152:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773848.233583:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773848.233594:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773848/real 1790773848] req@ffff9c4efbcc3100 x1877759534769152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773848.233605:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534769152/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773848.233614:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773848.233617:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773848.233621:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534769152:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773848.233627:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534769280:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773848.233642:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773848.233672:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773848/real 1790773848] req@ffff9c4efbcc1c00 x1877759534769280/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773848.233684:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534769280/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773848.233699:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773848.233702:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773848.233705:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534769280:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773848.233711:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534769408:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773848.233727:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773848.233737:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773848/real 1790773848] req@ffff9c4efbcc1f80 x1877759534769408/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773848.233747:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534769408/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773848.233755:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773848.233759:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773848.233762:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534769408:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773848.296744:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773848.296745:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773848.296745:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773848.296745:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773848.360741:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773848.360741:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773848.360805:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773848.488723:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773848.488723:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773848.552749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773848.616720:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773848.680741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773848.680741:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773848.680741:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773848.744769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773848.808745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773848.808746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773848.808746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773848.936747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773848.936747:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773848.936747:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773848.936748:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773849.000724:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773849.000724:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773849.000727:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773849.000732:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773849.000754:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773849.000757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773849.128703:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773849.192719:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773849.256740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773849.320753:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773849.320753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773849.320754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773849.320754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773849.384731:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773849.384736:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773849.384783:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773849.512749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773849.512749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773849.576741:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773849.640741:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773849.704747:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773849.704747:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773849.704758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773849.768733:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773849.832744:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773849.832744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773849.832744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773849.960751:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773849.960751:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773849.960751:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773849.960752:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773850.024716:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773850.024716:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773850.024728:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773850.024730:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773850.024749:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773850.024749:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773850.152739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773850.216756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773850.280715:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773850.344738:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773850.344738:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773850.344738:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773850.344738:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773850.408746:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773850.408762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773850.408829:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773850.536716:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773850.536716:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773850.600745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773850.664759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773850.728733:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773850.728739:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773850.728739:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773850.792766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773850.856747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773850.856747:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773850.856747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773850.984740:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773850.984740:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773850.984740:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773850.984786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773851.048770:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773851.048770:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773851.048780:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773851.048901:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773851.048907:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773851.048955:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773851.176767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773851.240746:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773851.304794:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773851.368808:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773851.368811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773851.368812:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773851.368812:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773851.432779:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773851.432779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:3.0:1790773851.432863:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773851.560736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773851.560764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773851.624778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773851.688781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773851.752768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773851.752768:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773851.752768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773851.816823:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773851.880780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773851.880780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773851.880786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773852.008757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773852.008757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773852.008757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773852.008793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773852.072735:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773852.072735:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773852.072736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773852.072745:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773852.072765:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773852.072771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773852.200760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773852.264756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773852.328731:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773852.392714:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773852.392717:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773852.392729:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773852.392747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773852.456735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773852.456798:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:3.0:1790773852.456877:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773852.584771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773852.584771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773852.648778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773852.712742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773852.776772:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773852.776778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773852.776870:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773852.840765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773852.904759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773852.904759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773852.904759:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773853.032757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773853.032757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773853.032757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773853.032774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773853.096736:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773853.096740:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773853.096743:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773853.096754:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773853.096768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773853.096782:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773853.224739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773853.288744:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773853.352715:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773853.352788:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773853.352794:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773853.352798:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773853.352807:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2849 00000100:00080000:0.0:1790773853.352816:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773853.352827:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773853.352861:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534769536/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773853.352886:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773853.352890:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773853.352892:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773853.352896:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2849 00000100:00080000:0.0:1790773853.352900:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773853.352905:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773853.352914:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773853.352932:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534769664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773853.352932:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534769536:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773853.352947:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773853.352950:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773853.352953:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773853.352957:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2849 00000100:00080000:0.0:1790773853.352961:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773853.352965:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773853.352970:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773853.352982:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534769792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773853.352992:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773853.352996:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773853.352998:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773853.353003:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2849 00000100:00080000:0.0:1790773853.353006:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773853.353013:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773853.353019:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773853.353026:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534769664:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773853.353034:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534769920/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773853.353047:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773853.353051:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773853.353054:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773853.353059:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534769792:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773853.353072:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2849 00000100:00080000:0.0:1790773853.353077:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773853.353086:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534769920:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773853.353087:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773853.353096:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773853.353112:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534770048/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773853.353124:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773853.353127:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773853.353129:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773853.353133:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534770048:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773853.353137:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2849 00000100:00080000:0.0:1790773853.353143:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773853.353150:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773853.353156:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773853.353171:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534770176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773853.353193:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773853.353204:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773853.353206:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773853.353211:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2849 00000100:00080000:0.0:1790773853.353216:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773853.353222:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773853.353227:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534770176:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773853.353228:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773853.353246:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534770304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773853.353258:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773853.353261:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1790773853.353264:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534770304:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773853.353265:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773853.353274:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2849 00000100:00080000:0.0:1790773853.353279:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773853.353286:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773853.353291:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773853.353303:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534770432/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773853.353321:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773853.353324:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773853.353327:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773853.353332:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2849 00000100:00080000:0.0:1790773853.353335:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773853.353339:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773853.353344:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773853.353348:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534770432:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773853.353354:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534770560/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773853.353382:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534770560:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773853.416744:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773853.416744:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773853.416750:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773853.416833:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773853.480739:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773853.480740:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773853.480833:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773853.608750:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773853.608750:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773853.672748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773853.736745:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773853.800713:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773853.800713:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773853.800715:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773853.864700:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773853.928711:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773853.928714:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773853.928718:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773854.056724:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773854.056724:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773854.056724:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773854.056724:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773854.120705:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773854.120716:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773854.120724:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773854.120725:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773854.120743:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773854.120752:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773854.248693:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773854.312758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773854.376704:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773854.440721:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773854.440734:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773854.440735:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773854.440816:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773854.504759:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773854.504968:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773854.504998:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773854.632745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773854.632749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773854.696743:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773854.760718:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773854.824703:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773854.824713:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773854.824713:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773854.888695:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773854.952746:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773854.952746:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773854.952746:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773855.080763:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773855.080763:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773855.080763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773855.080763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773855.144778:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773855.144778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773855.144836:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773855.144837:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773855.146496:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773855.146513:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773855.272765:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773855.336736:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773855.400727:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773855.464773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773855.464826:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773855.464893:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773855.465880:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773855.528742:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773855.528764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773855.528920:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773855.656754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773855.656754:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773855.720763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773855.784762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773855.848760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773855.848760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773855.848760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773855.912759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773855.976739:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773855.976739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773855.976749:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773856.104744:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773856.104744:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773856.104744:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773856.104853:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773856.168725:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773856.168736:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773856.168737:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773856.168754:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773856.168774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773856.168775:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773856.296733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773856.360758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773856.424757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:2.0:1790773856.424935:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:2.0:1790773856.424987:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1790773856.425008:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773856.425067:0:2384:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:2.0:1790773856.425081:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773856.425090:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773856.425098:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773856.425106:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773856.425117:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773856.425125:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:2.0:1790773856.425133:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:1.0:1790773856.425209:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773853/real 1790773856] req@ffff9c4ec7a7c700 x1877759534770560/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773856.425237:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534770560/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773856.425258:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773856.425264:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773856.425269:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534770560:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773856.425277:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773853/real 1790773856] req@ffff9c4ec7a7e300 x1877759534770432/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773856.425289:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534770432/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773856.425300:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773856.425318:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773856.425322:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534770432:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773856.425329:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773853/real 1790773856] req@ffff9c4ec7a7fb80 x1877759534770304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773856.425341:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534770304/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773856.425349:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773856.425352:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773856.425356:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534770304:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773856.425363:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773853/real 1790773856] req@ffff9c4ec7a7ca80 x1877759534770176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773856.425374:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534770176/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773856.425407:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773856.425411:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773856.425414:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534770176:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773856.425421:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773853/real 1790773856] req@ffff9c4ec7a7c380 x1877759534770048/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773856.425432:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534770048/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773856.425441:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773856.425445:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773856.425453:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534770048:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773856.425461:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773853/real 1790773856] req@ffff9c4ec7a7d880 x1877759534769920/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773856.425471:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534769920/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773856.425480:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773856.425484:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773856.425487:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534769920:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773856.425493:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773853/real 1790773856] req@ffff9c4ec7a7df80 x1877759534769664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773856.425505:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534769664/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773856.425521:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773856.425525:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773856.425529:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534769664:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773856.425541:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773853/real 1790773856] req@ffff9c4ec7a7ed80 x1877759534769792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773856.425552:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534769792/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773856.425560:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773856.425563:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773856.425566:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534769792:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773856.425572:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773853/real 1790773856] req@ffff9c4ec7a7f480 x1877759534769536/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773856.425583:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534769536/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773856.425589:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773856.425592:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773856.425595:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534769536:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773856.488766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773856.488766:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773856.488774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773856.488858:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773856.552720:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773856.552735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773856.552813:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773856.680722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773856.680722:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773856.744745:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773856.808765:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773856.872754:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773856.872754:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773856.872754:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773856.936764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773857.000748:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773857.000748:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773857.000749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773857.128756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773857.128756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773857.128756:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773857.128874:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773857.192732:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773857.192732:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773857.192738:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773857.192816:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773857.192835:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773857.192836:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773857.320756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773857.384753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773857.448760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773857.512756:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773857.512757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773857.512758:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773857.512849:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773857.576710:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773857.576834:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773857.576931:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773857.704768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773857.704769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773857.768782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773857.832775:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773857.896763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773857.896763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773857.896763:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773857.960772:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773858.024716:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773858.024723:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773858.024729:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773858.152764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773858.152770:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773858.152830:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773858.152885:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773858.216768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773858.216768:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773858.216769:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773858.216769:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773858.216798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773858.216798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773858.344735:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773858.408770:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773858.472753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773858.472814:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773858.472821:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773858.472825:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773858.472833:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2854 00000100:00080000:0.0:1790773858.472843:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773858.472854:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773858.472886:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534770688/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773858.472913:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773858.472917:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773858.472919:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773858.472923:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2854 00000100:00080000:0.0:1790773858.472926:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773858.472931:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773858.472940:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773858.472954:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534770816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773858.472964:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773858.472968:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773858.472970:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773858.472975:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2854 00000100:00080000:0.0:1790773858.472980:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773858.472986:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534770688:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773858.472987:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773858.472996:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773858.473008:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534770944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773858.473020:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773858.473024:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773858.473026:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773858.473030:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2854 00000100:00080000:0.0:1790773858.473045:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773858.473050:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773858.473056:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000800:00000100:1.0:1790773858.473057:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773858.473072:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534771072/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773858.473083:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773858.473086:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773858.473089:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773858.473093:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2854 00000100:00080000:0.0:1790773858.473097:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773858.473102:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773858/real 1790773858] req@ffff9c4ec7a7d180 x1877759534770688/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773858.473105:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773858.473113:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773858.473126:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534770688/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773858.473128:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534771200/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773858.473148:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773858.473149:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773858.473155:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773858.473156:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773858.473158:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773858.473163:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534770688:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773858.473167:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2854 00000100:00080000:0.0:1790773858.473173:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773858.473176:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534770816:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773858.473181:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773858.473188:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773858.473202:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534771328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773858.473207:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773858.473217:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773858.473221:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773858.473224:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773858.473227:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773858/real 1790773858] req@ffff9c4ec7a7f480 x1877759534770816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773858.473239:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2854 00000100:00080000:0.0:1790773858.473245:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773858.473248:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534770816/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773858.473253:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773858.473262:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773858.473274:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773858.473280:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773858.473284:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534770816:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773858.473292:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534771456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773858.473302:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773858.473305:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773858.473307:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773858.473312:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534770944:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773858.473315:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2854 00000100:00080000:0.0:1790773858.473323:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773858.473329:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773858.473334:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:1.0:1790773858.473353:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773858.473366:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534771584/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773858.473370:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773858/real 1790773858] req@ffff9c4ec7a7ed80 x1877759534770944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773858.473386:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773858.473391:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773858.473392:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534770944/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773858.473395:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773858.473404:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2854 00000100:00080000:0.0:1790773858.473411:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773858.473412:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773858.473417:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773858.473419:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790773858.473426:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773858.473426:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534770944:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773858.473438:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534771072:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773858.473445:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534771712/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773858.473466:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773858.473495:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773858/real 1790773858] req@ffff9c4ec7a7df80 x1877759534771072/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773858.473506:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534771072/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773858.473515:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773858.473518:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773858.473522:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534771072:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773858.473528:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534771200:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773858.473546:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773858.473557:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773858/real 1790773858] req@ffff9c4ec7a7d880 x1877759534771200/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773858.473568:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534771200/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773858.473577:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773858.473581:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773858.473584:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534771200:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773858.473590:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534771328:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773858.473609:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773858.473620:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773858/real 1790773858] req@ffff9c4ec7a7c380 x1877759534771328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773858.473629:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534771328/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773858.473643:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773858.473645:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773858.473649:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534771328:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773858.473687:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534771456:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773858.473706:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773858.473717:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773858/real 1790773858] req@ffff9c4ec7a7ca80 x1877759534771456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773858.473727:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534771456/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773858.473733:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773858.473736:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773858.473739:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534771456:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773858.473755:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534771584:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773858.473772:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773858.473781:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773858/real 1790773858] req@ffff9c4ec7a7fb80 x1877759534771584/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773858.473791:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534771584/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773858.473800:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773858.473803:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773858.473806:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534771584:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773858.473811:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534771712:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773858.473826:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773858.473835:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773858/real 1790773858] req@ffff9c4ec7a7e300 x1877759534771712/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773858.473844:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534771712/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773858.473853:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773858.473856:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773858.473860:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534771712:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773858.536754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773858.536754:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773858.536754:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773858.536754:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773858.600745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773858.600745:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773858.600861:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773858.728763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773858.728763:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773858.792757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773858.856773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773858.920802:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773858.920802:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773858.920929:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773858.984858:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773859.048765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773859.048765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773859.048765:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773859.176727:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773859.176731:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773859.176758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773859.176777:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773859.240808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773859.240827:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773859.241251:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773859.241546:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773859.242535:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773859.242558:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773859.368777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773859.432732:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773859.496756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773859.560815:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773859.560816:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773859.560816:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773859.560972:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773859.624823:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773859.624938:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773859.625047:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773859.752757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773859.753199:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773859.816821:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773859.880779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773859.944759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773859.944759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773859.944759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773860.008754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773860.072777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773860.072778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773860.072778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773860.200758:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773860.200758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773860.200765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773860.200766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773860.264765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773860.264765:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773860.264768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773860.264836:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773860.264848:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773860.264867:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773860.392760:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773860.456768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773860.520753:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773860.584755:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773860.584755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773860.584755:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773860.584781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773860.648765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773860.648768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773860.648908:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773860.776751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773860.776754:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773860.840775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773860.904804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773860.968799:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773860.968799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773860.968799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773861.032813:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773861.096778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773861.096782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773861.096782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773861.224810:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773861.224811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773861.224815:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773861.224987:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773861.288781:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773861.288863:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773861.288888:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773861.288905:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773861.288913:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773861.289023:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773861.416785:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773861.480869:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773861.545012:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773861.608810:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773861.608811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773861.608816:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773861.609319:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773861.672835:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773861.672938:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773861.673094:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773861.800756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773861.800918:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773861.864780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773861.928760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773861.992808:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773861.992808:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773861.993121:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773862.056755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773862.120773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773862.120773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773862.121021:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773862.248801:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773862.248801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773862.248801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773862.250925:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773862.312748:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773862.312792:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773862.312793:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773862.313185:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773862.313206:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773862.313230:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773862.440778:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773862.505047:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773862.568908:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773862.632716:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773862.632730:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773862.632739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773862.632837:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773862.698072:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773862.698170:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773862.699429:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773862.824776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773862.824996:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773862.888771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773862.952800:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773863.016771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773863.016855:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773863.016884:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773863.080855:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773863.144842:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773863.145478:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773863.146340:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773863.272770:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773863.272770:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773863.272880:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773863.273142:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773863.336990:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773863.337445:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773863.337473:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773863.337701:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773863.337722:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773863.337764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773863.464739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773863.528809:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773863.592803:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773863.592812:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773863.592817:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773863.592821:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773863.592830:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2859 00000100:00080000:1.0:1790773863.592839:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773863.592850:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773863.592884:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534771840/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773863.592920:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773863.592925:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773863.592927:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773863.592931:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2859 00000100:00080000:1.0:1790773863.592935:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773863.592940:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773863.592949:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773863.592962:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534771968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773863.592973:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773863.592976:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773863.592978:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773863.592982:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2859 00000100:00080000:1.0:1790773863.592986:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773863.592990:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773863.592995:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773863.593005:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534772096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773863.593014:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773863.593017:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773863.593019:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773863.593023:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2859 00000100:00080000:1.0:1790773863.593026:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773863.593031:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773863.593035:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773863.593045:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534772224/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773863.593053:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773863.593056:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773863.593057:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773863.593060:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2859 00000100:00080000:1.0:1790773863.593063:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773863.593066:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773863.593068:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773863.593074:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534772352/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773863.593080:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773863.593082:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773863.593083:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773863.593085:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2859 00000100:00080000:1.0:1790773863.593087:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773863.593089:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773863.593091:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773863.593096:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534772480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773863.593101:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773863.593103:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773863.593104:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773863.593107:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2859 00000100:00080000:1.0:1790773863.593108:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773863.593111:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773863.593114:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773863.593118:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534772608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773863.593124:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773863.593125:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773863.593126:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773863.593129:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2859 00000100:00080000:1.0:1790773863.593130:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773863.593132:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773863.593135:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773863.593139:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534772736/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773863.593144:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773863.593146:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773863.593147:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773863.593150:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2859 00000100:00080000:1.0:1790773863.593151:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773863.593154:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773863.593156:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773863.593161:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534772864/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773863.593283:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534771840:192.168.204.150@tcp:250:kworker.0 00000800:00000100:0.0:1790773863.593341:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773863.593378:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773863/real 1790773863] req@ffff9c4efac4df80 x1877759534771840/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773863.593395:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534771840/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773863.593404:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773863.593409:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773863.593413:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534771840:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773863.593419:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534771968:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773863.593440:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773863.593453:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773863/real 1790773863] req@ffff9c4efac4e300 x1877759534771968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773863.593465:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534771968/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773863.593493:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773863.593497:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773863.593501:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534771968:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773863.593507:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534772096:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773863.593526:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773863.593538:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773863/real 1790773863] req@ffff9c4efac4ea00 x1877759534772096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773863.593549:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534772096/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773863.593557:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773863.593560:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773863.593564:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534772096:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773863.593569:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534772224:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773863.593590:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773863.593601:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773863/real 1790773863] req@ffff9c4efac4fb80 x1877759534772224/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773863.593612:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534772224/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773863.593621:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773863.593623:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773863.593625:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534772224:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773863.593629:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534772352:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773863.593643:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773863.593669:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773863/real 1790773863] req@ffff9c4efac4dc00 x1877759534772352/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773863.593682:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534772352/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773863.593691:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773863.593695:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773863.593698:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534772352:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773863.593704:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534772480:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773863.593724:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773863.593734:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773863/real 1790773863] req@ffff9c4efac4e680 x1877759534772480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773863.593753:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534772480/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773863.593766:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773863.593768:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773863.593770:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534772480:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773863.593774:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534772608:192.168.204.150@tcp:38:kworker.0 00000800:00000100:0.0:1790773863.593837:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773863.593850:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773863/real 1790773863] req@ffff9c4efac4f480 x1877759534772608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773863.593862:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534772608/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773863.593869:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773863.593873:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773863.593876:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534772608:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773863.593882:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534772736:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773863.593902:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773863.593913:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773863/real 1790773863] req@ffff9c4efac4ed80 x1877759534772736/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773863.593924:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534772736/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773863.593939:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773863.593942:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773863.593946:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534772736:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773863.593952:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534772864:192.168.204.150@tcp:8:kworker.0 00000800:00000100:0.0:1790773863.593971:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773863.593981:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773863/real 1790773863] req@ffff9c4efac4c700 x1877759534772864/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773863.593992:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534772864/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773863.594001:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773863.594004:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773863.594008:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534772864:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773863.656759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773863.656759:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773863.656759:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773863.656823:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773863.721336:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773863.721528:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773863.721565:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773863.848765:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773863.848838:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773863.912786:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773863.976796:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773864.040776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773864.040776:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773864.040776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773864.105001:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773864.168760:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773864.168760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773864.168761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773864.296762:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773864.296762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773864.296971:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773864.297321:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773864.360789:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773864.361184:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773864.361274:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773864.361414:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773864.362021:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773864.362052:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773864.488775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773864.552817:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773864.616776:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773864.680764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773864.680776:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773864.681196:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773864.682306:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773864.744745:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773864.744779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773864.744861:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773864.872762:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773864.872762:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773864.936807:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773865.000772:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773865.064765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773865.064765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773865.064831:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773865.129110:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773865.192767:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773865.192767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773865.192793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773865.320750:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773865.320779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773865.320836:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773865.320878:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773865.384731:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773865.384731:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773865.384746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773865.384762:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773865.384779:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773865.384785:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773865.512777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773865.576869:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773865.640778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773865.704802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773865.704802:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773865.704814:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773865.704863:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773865.768841:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773865.768843:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773865.769038:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773865.896718:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773865.896763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773865.960772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773866.024742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773866.088795:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773866.088801:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773866.088802:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773866.152758:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773866.216778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773866.216778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773866.216865:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773866.344795:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773866.344800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773866.344801:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773866.344804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773866.408787:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773866.408787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773866.408787:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773866.408893:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773866.408906:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773866.409577:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773866.536759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773866.600793:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773866.664762:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773866.728731:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773866.728779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773866.728800:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773866.728828:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773866.792782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773866.792852:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773866.793011:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773866.920791:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773866.920792:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773866.984762:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773867.048743:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773867.112815:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773867.112816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773867.112818:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773867.176815:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773867.240742:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773867.240742:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773867.240745:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773867.368745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773867.368746:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773867.368746:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773867.368775:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773867.432716:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773867.432726:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773867.432726:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773867.432730:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773867.432760:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773867.432772:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773867.560739:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773867.624768:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773867.688774:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773867.752763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773867.752763:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773867.752786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773867.752796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773867.816730:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773867.816772:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773867.816822:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773867.944717:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773867.944741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773868.008772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773868.072747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773868.136756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773868.136756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773868.136756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773868.200794:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773868.264756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773868.264756:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773868.264757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773868.392837:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773868.392837:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773868.393221:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773868.393229:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773868.456753:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773868.456795:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773868.456821:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773868.456909:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773868.456990:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773868.459705:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773868.584769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773868.648789:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773868.712837:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773868.715933:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773868.715941:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773868.715945:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773868.715955:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2864 00000100:00080000:2.0:1790773868.715965:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773868.715977:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773868.716011:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534772992/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773868.716037:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773868.716041:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773868.716043:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773868.716047:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2864 00000100:00080000:2.0:1790773868.716050:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773868.716054:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773868.716062:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773868.716072:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534773120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773868.716081:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773868.716084:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773868.716086:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773868.716090:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2864 00000100:00080000:2.0:1790773868.716093:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773868.716097:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773868.716102:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773868.716111:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534772992:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773868.716113:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534773248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773868.716125:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773868.716128:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773868.716130:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773868.716135:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2864 00000100:00080000:2.0:1790773868.716138:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773868.716155:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773868.716160:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773868.716171:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534773376/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773868.716181:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773868.716183:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773868.716187:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773868.716190:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773868.716195:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2864 00000100:00080000:2.0:1790773868.716200:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773868.716205:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773868.716210:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773868.716221:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534773504/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773868.716227:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773868/real 1790773868] req@ffff9c4efbcc2300 x1877759534772992/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773868.716234:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773868.716247:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1790773868.716249:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534772992/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773868.716250:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773868.716261:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2864 00000100:00080000:1.0:1790773868.716265:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773868.716267:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773868.716273:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773868.716274:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773868.716281:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534772992:192.168.204.150@tcp:250:kworker.0 00000008:00100000:2.0:1790773868.716281:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773868.716296:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534773632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773868.716299:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534773120:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773868.716312:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773868.716315:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773868.716317:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773868.716321:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2864 00000100:00080000:2.0:1790773868.716325:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773868.716332:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773868.716334:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:2.0:1790773868.716337:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773868.716346:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534773760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773868.716350:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773868/real 1790773868] req@ffff9c4efbcc0380 x1877759534773120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773868.716356:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773868.716362:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773868.716365:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773868.716373:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2864 00000100:00080000:2.0:1790773868.716379:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773868.716385:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534773120/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773868.716386:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773868.716394:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773868.716407:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1790773868.716408:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534773888/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773868.716415:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773868.716422:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534773120:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773868.716427:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773868.716431:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773868.716433:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773868.716439:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2864 00000100:00080000:2.0:1790773868.716442:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773868.716445:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773868.716451:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773868.716452:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534773248:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773868.716466:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534774016/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773868.716477:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773868.716491:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773868/real 1790773868] req@ffff9c4efbcc1f80 x1877759534773248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773868.716504:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534773248/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773868.716513:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773868.716518:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773868.716523:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534773248:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773868.716530:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534773376:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773868.716547:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773868.716555:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773868/real 1790773868] req@ffff9c4efbcc1c00 x1877759534773376/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773868.716565:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534773376/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773868.716581:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773868.716585:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773868.716589:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534773376:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773868.716607:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534773504:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773868.716624:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773868.716633:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773868/real 1790773868] req@ffff9c4efbcc3100 x1877759534773504/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773868.716644:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534773504/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773868.716675:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773868.716679:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773868.716683:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534773504:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773868.716689:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534773632:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773868.716707:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773868.716716:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773868/real 1790773868] req@ffff9c4efbcc2680 x1877759534773632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773868.716739:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534773632/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773868.716754:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773868.716756:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773868.716760:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534773632:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773868.716766:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534773760:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773868.716781:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773868.716790:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773868/real 1790773868] req@ffff9c4efbcc1880 x1877759534773760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773868.716801:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534773760/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773868.716809:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773868.716812:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773868.716815:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534773760:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773868.716820:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534773888:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773868.716834:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773868.716842:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773868/real 1790773868] req@ffff9c4efbcc1500 x1877759534773888/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773868.716856:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534773888/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773868.716866:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773868.716869:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773868.716872:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534773888:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773868.716890:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534774016:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773868.716916:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773868.716926:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773868/real 1790773868] req@ffff9c4efbcc0700 x1877759534774016/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773868.716938:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534774016/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773868.716947:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773868.716950:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773868.716953:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534774016:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773868.776788:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773868.776795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773868.776821:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773868.777506:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773868.841276:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773868.841305:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773868.841329:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773868.968779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773868.968779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773869.032757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773869.096737:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773869.160802:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773869.160803:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773869.160803:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773869.224765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773869.288755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773869.288755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773869.288856:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773869.416780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773869.416780:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773869.416783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773869.416925:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773869.480810:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773869.480810:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773869.480815:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773869.481079:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773869.481094:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773869.482098:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773869.608730:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773869.672805:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773869.736739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773869.800761:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773869.800761:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773869.800761:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773869.800774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773869.864764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773869.864796:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773869.864838:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773869.992800:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773869.992815:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773870.056776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773870.121691:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773870.184717:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773870.184718:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773870.184735:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773870.248734:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773870.312765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773870.312765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773870.312765:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773870.440731:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773870.440739:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773870.440751:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773870.440784:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773870.504765:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773870.504765:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773870.504765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773870.504797:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773870.504824:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773870.504825:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773870.632776:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773870.696833:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773870.760909:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773870.824771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773870.824772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773870.824778:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773870.824797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773870.888805:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773870.888846:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773870.888882:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773871.016724:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773871.016724:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773871.080767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773871.144742:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773871.208763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773871.208763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773871.208766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773871.272794:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773871.336795:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773871.336796:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773871.336798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773871.464778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773871.464778:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773871.464849:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773871.464975:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773871.528822:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773871.528822:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773871.528822:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773871.528889:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773871.528894:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773871.528923:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773871.656927:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773871.720801:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773871.785749:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773871.848764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773871.848764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773871.848789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773871.848789:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773871.912768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773871.912805:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773871.912880:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773872.040830:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773872.041023:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773872.104773:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773872.168783:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773872.232782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773872.232782:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773872.232782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773872.296750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773872.360920:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773872.360954:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773872.361126:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773872.488762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773872.488762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773872.488782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773872.488900:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773872.552800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773872.552800:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773872.552822:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773872.552847:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773872.552862:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773872.552875:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773872.680791:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773872.744831:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773872.808734:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773872.872784:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773872.872784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773872.872795:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773872.872796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773872.936835:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773872.937055:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773872.937140:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773873.064820:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773873.064821:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773873.128766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773873.192792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773873.256756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773873.256756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773873.256756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773873.320727:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773873.384775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773873.384777:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773873.384778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773873.512827:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773873.512828:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773873.512829:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773873.512961:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773873.576816:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773873.576816:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773873.576816:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773873.576817:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773873.576853:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773873.576854:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773873.704781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773873.768852:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773873.832828:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773873.832972:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773873.832978:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773873.832983:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773873.832993:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2869 00000100:00080000:3.0:1790773873.833003:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773873.833015:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773873.833061:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534774144/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773873.833093:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773873.833100:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773873.833102:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773873.833108:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2869 00000100:00080000:3.0:1790773873.833112:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773873.833118:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773873.833128:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534774144:192.168.204.150@tcp:250:kworker.0 00000008:00100000:3.0:1790773873.833129:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773873.833148:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534774272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773873.833165:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773873.833168:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773873.833170:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773873.833175:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2869 00000100:00080000:3.0:1790773873.833180:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773873.833185:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773873.833189:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790773873.833191:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773873.833203:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534774400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773873.833212:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773873.833216:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773873.833217:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773873.833221:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2869 00000100:00080000:3.0:1790773873.833225:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773873.833230:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773873.833235:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773873.833236:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773873/real 1790773873] req@ffff9c4ef73c0000 x1877759534774144/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773873.833250:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534774528/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773873.833255:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534774144/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773873.833269:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773873.833273:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773873.833275:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773873.833280:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773873.833281:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2869 00000100:00080000:3.0:1790773873.833285:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773873.833291:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773873.833293:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773873.833299:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534774144:192.168.204.150@tcp:250:kworker.0 00000008:00100000:3.0:1790773873.833300:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773873.833316:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534774656/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773873.833327:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534774272:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773873.833327:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773873.833334:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773873.833338:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773873.833343:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2869 00000100:00080000:3.0:1790773873.833347:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773873.833354:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790773873.833358:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790773873.833362:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773873.833374:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773873/real 1790773873] req@ffff9c4ef73c1500 x1877759534774272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773873.833377:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534774784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773873.833401:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773873.833407:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773873.833409:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773873.833413:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534774272/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773873.833417:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2869 00000100:00080000:3.0:1790773873.833422:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773873.833430:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773873.833436:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1790773873.833440:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773873.833446:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790773873.833449:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534774912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773873.833453:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534774272:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773873.833467:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534774400:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773873.833480:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773873.833484:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773873.833486:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773873.833491:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2869 00000100:00080000:3.0:1790773873.833497:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790773873.833502:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773873.833503:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773873.833510:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773873.833516:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773873/real 1790773873] req@ffff9c4ef73c0700 x1877759534774400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773873.833524:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534775040/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773873.833529:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534774400/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773873.833540:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773873.833542:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773873.833544:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534774400:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773873.833548:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534774528:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773873.833567:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773873.833581:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773873/real 1790773873] req@ffff9c4ef73c2a00 x1877759534774528/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773873.833593:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534774528/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773873.833609:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773873.833615:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773873.833616:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773873.833617:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773873.833624:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773873.833626:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2869 00000100:00100000:2.0:1790773873.833631:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534774528:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790773873.833634:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773873.833643:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773873.833648:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773873.833672:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534774656:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773873.833693:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773873.833704:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534775168/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773873.833710:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773873/real 1790773873] req@ffff9c4ef73c2d80 x1877759534774656/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773873.833730:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534774656/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773873.833739:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773873.833743:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773873.833747:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534774656:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773873.833751:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534774784:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773873.833766:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773873.833774:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773873/real 1790773873] req@ffff9c4ef73c3800 x1877759534774784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773873.833784:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534774784/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773873.833807:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773873.833811:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773873.833815:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534774784:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773873.833821:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534774912:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773873.833840:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773873.833852:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773873/real 1790773873] req@ffff9c4ef73c0a80 x1877759534774912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773873.833863:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534774912/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773873.833871:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773873.833875:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773873.833878:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534774912:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773873.833884:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534775040:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773873.833902:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773873.833914:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773873/real 1790773873] req@ffff9c4ef73c0e00 x1877759534775040/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773873.833923:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534775040/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773873.833934:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773873.833937:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773873.833940:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534775040:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773873.833958:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534775168:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773873.833975:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773873.833984:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773873/real 1790773873] req@ffff9c4ef73c2300 x1877759534775168/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773873.833991:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534775168/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773873.833997:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773873.833999:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773873.834001:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534775168:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773873.896760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773873.896760:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773873.896761:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773873.896874:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773873.960739:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773873.960762:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773873.960797:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773874.088777:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773874.088777:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773874.152753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773874.216966:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773874.280771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773874.280771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773874.280884:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773874.344756:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773874.408768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773874.408768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773874.408768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773874.536766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773874.536771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773874.536771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773874.536775:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773874.600752:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773874.600755:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773874.600775:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773874.600883:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773874.600895:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773874.600912:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773874.728791:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773874.792800:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773874.857133:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773874.920775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773874.920775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773874.920779:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773874.920801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773874.984753:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773874.984779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773874.984832:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773875.112738:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773875.112741:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773875.176790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773875.240779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773875.304756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773875.304756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773875.304868:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773875.368820:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773875.432766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773875.432766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773875.432830:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773875.560790:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773875.560797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773875.560797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773875.560937:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773875.624835:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773875.624984:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773875.625109:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773875.625139:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773875.625157:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773875.625298:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773875.752769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773875.816811:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773875.880752:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773875.944803:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773875.944803:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773875.944809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773875.944955:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773876.008826:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773876.008897:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773876.008967:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773876.136747:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773876.136767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773876.200770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773876.264907:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773876.328751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773876.328751:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773876.328840:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773876.392798:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773876.456771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773876.456771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773876.456773:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773876.584768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773876.584768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773876.584768:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773876.584899:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773876.648805:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773876.648831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773876.648833:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773876.648875:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773876.648875:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773876.648893:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773876.776852:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773876.840803:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773876.904843:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773876.968758:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773876.968758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773876.968758:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773876.969019:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773877.032770:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773877.032802:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773877.032813:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773877.160775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773877.160779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773877.224790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773877.288778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773877.352768:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773877.352768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773877.352873:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773877.416819:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773877.480761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773877.480761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773877.480762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773877.608790:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773877.608790:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773877.608794:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773877.608799:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773877.672803:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773877.672804:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773877.672813:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773877.672813:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773877.672856:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773877.672860:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773877.800751:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773877.864819:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773877.928789:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773877.992794:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773877.992797:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773877.992808:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773877.992810:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773878.056775:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773878.056784:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773878.056839:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773878.184818:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773878.184855:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773878.248778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773878.312788:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773878.376765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773878.376765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773878.376765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773878.440764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773878.504768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773878.504768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773878.504786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773878.632783:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773878.632783:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773878.632783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773878.632800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773878.696788:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773878.696835:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773878.696839:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773878.696996:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773878.697083:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773878.698159:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773878.824755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773878.888804:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773878.952902:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773878.952911:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773878.952922:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773878.952928:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773878.952940:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2874 00000100:00080000:1.0:1790773878.952965:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773878.952978:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773878.953018:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534775296/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773878.953050:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773878.953055:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773878.953057:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773878.953062:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2874 00000100:00080000:1.0:1790773878.953066:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773878.953071:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773878.953082:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773878.953099:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534775424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773878.953112:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773878.953113:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534775296:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773878.953123:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773878.953126:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773878.953132:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2874 00000100:00080000:1.0:1790773878.953146:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773878.953153:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773878.953159:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773878.953174:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534775552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773878.953184:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773878.953189:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773878.953192:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773878.953196:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2874 00000100:00080000:1.0:1790773878.953201:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773878.953210:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790773878.953213:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:1.0:1790773878.953219:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773878.953235:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534775680/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773878.953247:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773878.953250:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773878.953252:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773878.953257:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2874 00000100:00080000:1.0:1790773878.953263:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773878.953272:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773878.953280:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773878.953281:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773878/real 1790773878] req@ffff9c4efac4f800 x1877759534775296/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773878.953299:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534775808/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773878.953308:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534775296/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773878.953314:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773878.953326:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773878.953329:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773878.953330:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773878.953345:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773878.953346:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2874 00000100:00080000:1.0:1790773878.953358:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773878.953358:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534775296:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773878.953371:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773878.953378:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773878.953391:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534775936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773878.953400:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534775424:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773878.953412:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773878.953418:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773878.953421:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773878.953428:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2874 00000100:00080000:1.0:1790773878.953434:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790773878.953441:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773878.953445:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773878.953453:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773878.953464:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773878/real 1790773878] req@ffff9c4efac4c700 x1877759534775424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773878.953471:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534776064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773878.953492:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773878.953494:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534775424/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773878.953500:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773878.953504:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773878.953517:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2874 00000100:00080000:1.0:1790773878.953523:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773878.953530:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773878.953532:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773878.953542:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:1.0:1790773878.953544:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773878.953552:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534775424:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773878.953562:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534776192/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773878.953565:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534775552:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773878.953584:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773878.953588:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773878.953591:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773878.953597:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2874 00000800:00000100:2.0:1790773878.953604:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773878.953607:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773878.953617:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773878.953624:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773878.953628:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773878/real 1790773878] req@ffff9c4efac4ed80 x1877759534775552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773878.953640:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534776320/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773878.953647:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534775552/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773878.953714:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773878.953719:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773878.953723:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534775552:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773878.953731:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534775680:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773878.953756:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773878.953768:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773878/real 1790773878] req@ffff9c4efac4f480 x1877759534775680/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773878.953780:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534775680/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773878.953794:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773878.953798:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773878.953802:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534775680:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773878.953808:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534775808:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773878.953825:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773878.953834:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773878/real 1790773878] req@ffff9c4efac4e680 x1877759534775808/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773878.953845:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534775808/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773878.953854:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773878.953858:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773878.953861:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534775808:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773878.953881:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534775936:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773878.953900:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773878.953910:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773878/real 1790773878] req@ffff9c4efac4dc00 x1877759534775936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773878.953921:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534775936/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773878.953937:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773878.953941:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773878.953959:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534775936:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773878.953966:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534776064:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773878.953984:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773878.953995:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773878/real 1790773878] req@ffff9c4efac4fb80 x1877759534776064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773878.954006:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534776064/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773878.954014:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773878.954018:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773878.954021:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534776064:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773878.954027:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534776192:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773878.954044:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773878.954053:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773878/real 1790773878] req@ffff9c4efac4ea00 x1877759534776192/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773878.954063:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534776192/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773878.954072:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773878.954076:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773878.954079:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534776192:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773878.954084:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534776320:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773878.954098:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773878.954107:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773878/real 1790773878] req@ffff9c4efac4e300 x1877759534776320/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773878.954116:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534776320/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773878.954125:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773878.954128:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773878.954131:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534776320:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773879.016795:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773879.016795:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773879.016808:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773879.016813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773879.080825:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773879.080865:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773879.080884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773879.208820:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773879.208845:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773879.272797:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773879.336830:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773879.400782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773879.400782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773879.400785:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773879.464855:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773879.528777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773879.528777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773879.528824:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773879.656798:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773879.656798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773879.656944:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773879.656977:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773879.720734:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773879.720763:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773879.720774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773879.720779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773879.720797:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773879.720798:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773879.848767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773879.912796:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773879.976902:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773880.040798:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773880.040820:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773880.040853:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773880.040876:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773880.104795:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773880.104795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773880.105076:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773880.232796:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773880.232820:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773880.296774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773880.360771:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773880.425033:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773880.425033:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773880.425144:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773880.488736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773880.552796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773880.553383:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773880.553735:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773880.680782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773880.680782:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773880.680900:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773880.680985:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773880.744813:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773880.744821:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773880.744823:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773880.744823:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773880.744854:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773880.744861:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773880.872791:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773880.936765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773881.000746:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773881.064809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773881.064810:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773881.064810:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773881.064816:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773881.128800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773881.128832:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773881.128861:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773881.256744:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773881.256760:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773881.320788:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773881.384805:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773881.448806:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773881.448806:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773881.448807:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773881.512806:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773881.576788:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773881.576789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773881.576794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773881.704755:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773881.704756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773881.704765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773881.705370:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773881.768816:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773881.768818:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773881.768826:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773881.768853:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773881.768856:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773881.768903:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773881.896753:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773881.960777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773882.024761:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773882.088799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773882.088804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773882.088809:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773882.088946:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773882.152791:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773882.152799:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773882.152874:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773882.280788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773882.280788:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773882.344767:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773882.408769:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773882.472770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773882.472920:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773882.474012:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773882.536792:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773882.600737:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773882.600740:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773882.600832:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773882.728779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773882.728783:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773882.728787:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773882.728825:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773882.792784:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773882.792789:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773882.792790:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773882.792970:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773882.793010:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773882.793016:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773882.920801:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773882.984802:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773883.048821:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773883.112792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773883.112792:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773883.112792:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773883.112886:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00010000:00010000:1.0:1790773883.176832:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773883.176895:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773883.176947:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773883.304710:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773883.304766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773883.368785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773883.432815:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773883.496764:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773883.496764:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773883.496837:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773883.562123:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773883.624821:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773883.624951:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773883.625034:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773883.752819:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773883.752819:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773883.752841:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773883.752842:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773883.816798:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773883.816816:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773883.816817:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773883.816907:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773883.816931:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773883.816959:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773883.945018:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773884.008737:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773884.072792:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773884.072805:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773884.072811:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773884.072824:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2879 00000100:00080000:0.0:1790773884.072837:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773884.072864:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773884.072897:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773884.072919:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534776448/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773884.072951:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773884.072955:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773884.072958:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773884.072964:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2879 00000100:00080000:0.0:1790773884.072970:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773884.072976:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773884.072984:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534776448:192.168.204.150@tcp:250:kworker.0 00000008:00100000:0.0:1790773884.072990:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773884.073019:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534776576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773884.073033:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773884.073037:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773884.073040:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773884.073046:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2879 00000100:00080000:0.0:1790773884.073050:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790773884.073052:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773884.073064:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773884.073072:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773884.073099:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534776704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773884.073110:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773884/real 1790773884] req@ffff9c4ec7a7c700 x1877759534776448/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773884.073113:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773884.073126:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773884.073130:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773884.073138:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2879 00000100:00080000:0.0:1790773884.073153:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773884.073165:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773884.073166:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534776448/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773884.073175:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00080000:2.0:1790773884.073190:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773884.073195:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534776832/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773884.073200:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773884.073216:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534776448:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773884.073221:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773884.073226:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773884.073229:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773884.073235:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2879 00000100:00080000:0.0:1790773884.073239:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773884.073246:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773884.073253:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534776576:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773884.073255:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773884.073277:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534776960/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773884.073290:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790773884.073300:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773884.073310:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773884.073312:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773884.073321:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2879 00000100:00100000:2.0:1790773884.073323:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773884/real 1790773884] req@ffff9c4ec7a7e300 x1877759534776576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773884.073330:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773884.073345:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773884.073351:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534776576/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773884.073356:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773884.073386:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534777088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773884.073387:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773884.073400:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:0.0:1790773884.073411:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773884.073416:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773884.073419:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773884.073420:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534776576:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773884.073428:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2879 00000100:00080000:0.0:1790773884.073435:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773884.073437:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534776704:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773884.073444:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773884.073451:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773884.073466:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534777216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790773884.073475:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773884.073489:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773884.073494:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773884.073496:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773884.073502:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773884/real 1790773884] req@ffff9c4ec7a7fb80 x1877759534776704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773884.073506:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2879 00000100:00080000:0.0:1790773884.073513:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773884.073523:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773884.073530:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534776704/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773884.073539:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:2.0:1790773884.073543:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773884.073549:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773884.073558:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534777344/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773884.073558:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534776704:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773884.073583:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773884.073587:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534776832:192.168.204.150@tcp:8:kworker.0 00000100:00080000:0.0:1790773884.073597:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773884.073600:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773884.073607:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2879 00000100:00080000:0.0:1790773884.073612:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1790773884.073618:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773884.073623:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773884.073630:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773884.073638:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773884/real 1790773884] req@ffff9c4ec7a7ca80 x1877759534776832/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773884.073647:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534777472/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773884.073694:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534776832/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773884.073707:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773884.073712:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773884.073718:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534776832:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773884.073737:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534776960:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773884.073757:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773884.073768:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773884/real 1790773884] req@ffff9c4ec7a7c380 x1877759534776960/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773884.073775:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534776960/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773884.073781:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773884.073784:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773884.073786:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534776960:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773884.073790:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534777088:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773884.073807:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773884.073817:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773884/real 1790773884] req@ffff9c4ec7a7d880 x1877759534777088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773884.073828:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534777088/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773884.073858:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773884.073862:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773884.073865:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534777088:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773884.073872:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534777216:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773884.073892:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773884.073903:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773884/real 1790773884] req@ffff9c4ec7a7df80 x1877759534777216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773884.073910:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534777216/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773884.073915:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773884.073918:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773884.073920:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534777216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773884.073924:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534777344:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773884.073936:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773884.073942:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773884/real 1790773884] req@ffff9c4ec7a7ed80 x1877759534777344/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773884.073948:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534777344/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773884.073954:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773884.073956:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773884.073959:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534777344:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773884.073962:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534777472:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773884.073971:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773884.073977:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773884/real 1790773884] req@ffff9c4ec7a7f480 x1877759534777472/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773884.073983:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534777472/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773884.073989:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773884.073991:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773884.073992:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534777472:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773884.136813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773884.136813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773884.136830:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773884.136882:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773884.200802:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773884.200882:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773884.200910:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773884.328751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773884.328751:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773884.392766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773884.456761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773884.520770:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773884.520770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773884.520773:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773884.584783:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773884.648755:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773884.648755:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773884.648755:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773884.776749:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773884.776749:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773884.776749:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773884.776749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773884.840743:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773884.840744:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773884.840745:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773884.840758:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773884.840774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773884.840806:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773884.968910:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773885.032792:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773885.096814:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773885.160769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773885.160787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773885.160806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773885.160840:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773885.224799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773885.224855:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773885.224936:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773885.352768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773885.352804:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773885.416770:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773885.480759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773885.544759:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773885.544759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773885.544759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773885.608755:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773885.672765:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773885.672765:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773885.672870:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773885.800758:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773885.800758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773885.800788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773885.800873:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773885.864756:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773885.864763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773885.864788:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773885.864790:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773885.864824:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773885.864854:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773885.992980:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773886.056760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773886.120821:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773886.184787:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773886.184787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773886.184787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773886.184826:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773886.248788:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773886.248975:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773886.249033:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773886.376782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773886.376788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773886.440716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773886.504780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773886.568783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773886.568905:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773886.568989:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773886.632752:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773886.696758:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773886.696758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773886.696923:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773886.824833:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773886.824834:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773886.824836:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773886.824911:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773886.888766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773886.888776:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773886.888808:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773886.888828:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773886.888854:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773886.888854:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773887.016776:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773887.080738:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773887.144777:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773887.208772:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773887.208772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773887.208772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773887.208878:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00010000:00010000:1.0:1790773887.272828:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773887.273235:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773887.273251:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773887.400829:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773887.400942:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773887.464765:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773887.528790:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773887.592727:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773887.592727:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773887.592730:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773887.656825:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773887.720802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773887.720802:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773887.720838:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773887.848800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773887.848800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773887.848801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773887.848813:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773887.912801:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773887.912839:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773887.912840:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773887.912840:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773887.912862:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773887.912881:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773888.040824:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773888.104779:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773888.168789:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773888.232793:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773888.232793:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773888.232832:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773888.232903:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773888.296793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773888.297019:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773888.297116:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773888.424773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773888.424773:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773888.488821:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773888.552790:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773888.616792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773888.616792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773888.616798:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773888.680894:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773888.744966:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773888.745230:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773888.745257:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773888.872860:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773888.872949:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773888.873143:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773888.873261:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773888.936883:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773888.936884:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773888.937168:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773888.937197:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773888.937213:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773888.937316:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773889.064915:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773889.128942:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773889.192879:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773889.192917:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773889.192922:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773889.192927:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773889.192936:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2885 00000100:00080000:2.0:1790773889.192946:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773889.192958:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773889.192997:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534777600/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773889.193023:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773889.193027:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773889.193030:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773889.193036:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2885 00000100:00080000:2.0:1790773889.193041:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773889.193050:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773889.193056:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534777600:192.168.204.150@tcp:250:kworker.0 00000008:00100000:2.0:1790773889.193062:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773889.193077:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534777728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773889.193087:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773889.193090:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773889.193092:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773889.193096:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2885 00000100:00080000:2.0:1790773889.193099:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773889.193104:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773889.193108:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773889.193118:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773889.193120:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534777856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773889.193129:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773889.193133:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773889.193135:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773889.193139:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2885 00000100:00080000:2.0:1790773889.193142:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773889.193147:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773889.193151:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773889.193161:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534777984/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773889.193171:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773889.193174:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773889.193177:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773889.193178:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773889/real 1790773889] req@ffff9c4efbcc2a00 x1877759534777600/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773889.193184:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2885 00000100:00080000:2.0:1790773889.193190:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773889.193197:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773889.193203:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773889.193207:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534777600/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773889.193216:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534778112/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773889.193223:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773889.193228:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773889.193231:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773889.193233:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773889.193237:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773889.193240:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534777600:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773889.193244:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2885 00000100:00080000:2.0:1790773889.193250:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773889.193256:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773889.193260:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773889.193271:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534778240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773889.193277:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534777728:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773889.193288:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773889.193292:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773889.193295:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773889.193301:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2885 00000100:00080000:2.0:1790773889.193305:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773889.193310:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773889.193313:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773889.193319:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773889.193327:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773889/real 1790773889] req@ffff9c4efbcc0a80 x1877759534777728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773889.193333:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534778368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773889.193350:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534777728/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773889.193351:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773889.193358:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773889.193362:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773889.193374:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2885 00000100:00080000:1.0:1790773889.193377:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773889.193379:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773889.193385:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773889.193389:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773889.193394:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534777728:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790773889.193397:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773889.193404:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534777856:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773889.193413:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534778496/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773889.193424:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773889.193430:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773889.193431:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773889.193436:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773889.193442:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2885 00000100:00080000:2.0:1790773889.193446:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773889.193448:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773889/real 1790773889] req@ffff9c4efbcc0700 x1877759534777856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773889.193454:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773889.193463:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773889.193471:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534777856/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773889.193480:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773889.193483:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773889.193486:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534777856:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773889.193492:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534777984:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773889.193510:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773889.193520:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773889/real 1790773889] req@ffff9c4efbcc1500 x1877759534777984/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773889.193538:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534777984/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773889.193538:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534778624/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773889.193582:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773889.193588:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773889.193593:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534777984:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773889.193600:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534778112:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773889.193626:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773889.193637:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773889/real 1790773889] req@ffff9c4efbcc1880 x1877759534778112/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773889.193689:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534778112/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773889.193698:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773889.193702:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773889.193709:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534778112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773889.193727:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534778240:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773889.193760:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773889.193773:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773889/real 1790773889] req@ffff9c4efbcc2680 x1877759534778240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773889.193785:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534778240/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773889.193801:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773889.193805:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773889.193808:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534778240:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773889.193813:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534778368:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773889.193828:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773889.193835:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773889/real 1790773889] req@ffff9c4efbcc3100 x1877759534778368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773889.193842:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534778368/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773889.193851:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773889.193854:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773889.193856:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534778368:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773889.193860:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534778496:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773889.193869:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773889.193877:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773889/real 1790773889] req@ffff9c4efbcc1c00 x1877759534778496/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773889.193888:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534778496/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773889.193899:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773889.193902:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773889.193906:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534778496:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773889.193911:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534778624:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773889.193930:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773889.193940:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773889/real 1790773889] req@ffff9c4efbcc1f80 x1877759534778624/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773889.193951:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534778624/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773889.193961:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773889.193964:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773889.193968:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534778624:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773889.256777:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773889.256836:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773889.256847:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773889.258038:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773889.320784:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773889.320859:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773889.320946:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773889.448850:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773889.448907:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773889.512816:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773889.576787:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773889.640813:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773889.640813:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773889.640867:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773889.704774:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773889.768776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773889.768776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773889.768883:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773889.896799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773889.896800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773889.896800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773889.896809:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773889.960792:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773889.960816:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773889.960844:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773889.960854:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773889.960858:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773889.960886:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773890.088776:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773890.152815:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773890.216779:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773890.280802:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773890.280803:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773890.280803:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773890.280803:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00010000:00010000:2.0:1790773890.344936:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773890.345034:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773890.345097:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773890.472780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773890.472795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773890.536838:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773890.600815:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773890.664775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773890.664775:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773890.664787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773890.728782:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773890.792744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773890.792744:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773890.792773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773890.920784:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773890.920784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773890.920784:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773890.920871:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773890.984778:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773890.984805:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773890.984806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773890.984806:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773890.984828:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773890.984859:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773891.112748:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773891.176774:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773891.240784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773891.304768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773891.304768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773891.304768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773891.304768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773891.368742:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773891.368745:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773891.368832:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773891.496742:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773891.496752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773891.560760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773891.624939:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773891.688778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773891.688779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773891.688800:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773891.752783:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773891.816770:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773891.816770:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773891.816821:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773891.944764:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773891.944764:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773891.944764:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773891.944800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773892.008779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773892.008784:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773892.008804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773892.008832:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773892.008906:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773892.008924:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773892.136812:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773892.200792:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773892.264779:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773892.328796:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773892.328804:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773892.328804:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773892.328806:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773892.392780:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773892.392822:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773892.392870:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773892.520747:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773892.520763:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773892.584784:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773892.648786:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773892.712767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773892.712767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773892.712767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773892.776800:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773892.840739:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773892.840749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773892.840767:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773892.968760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773892.968760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773892.968760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773892.968760:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773893.032773:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773893.032773:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773893.032773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773893.032903:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773893.032931:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773893.032933:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773893.160775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773893.224815:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773893.288765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773893.352829:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773893.352829:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773893.352834:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773893.352951:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773893.416812:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773893.416820:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773893.416860:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773893.544792:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773893.544944:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773893.608758:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773893.672794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773893.736771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773893.736771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773893.736771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773893.800773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773893.864808:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773893.864816:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773893.864891:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773893.992771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773893.992771:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773893.992771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773893.992772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773894.056831:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773894.056847:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773894.056849:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773894.056851:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773894.056878:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773894.056880:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773894.184867:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773894.248764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773894.312784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773894.312788:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773894.312797:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773894.312802:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773894.312813:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2890 00000100:00080000:3.0:1790773894.312823:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773894.312836:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773894.312874:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534778752/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773894.312916:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773894.312920:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773894.312922:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773894.312927:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2890 00000100:00080000:3.0:1790773894.312930:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773894.312935:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773894.312946:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773894.312960:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534778880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773894.312970:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773894.312974:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773894.312976:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773894.312988:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2890 00000100:00080000:3.0:1790773894.312993:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773894.312997:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773894.313002:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773894.313013:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534779008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773894.313023:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773894.313027:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773894.313029:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773894.313033:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2890 00000100:00080000:3.0:1790773894.313036:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773894.313040:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773894.313045:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773894.313054:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534779136/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773894.313063:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773894.313067:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773894.313069:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773894.313073:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2890 00000100:00080000:3.0:1790773894.313076:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773894.313080:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773894.313085:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773894.313107:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534779264/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773894.313118:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773894.313122:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773894.313124:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773894.313128:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2890 00000100:00080000:3.0:1790773894.313132:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773894.313137:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773894.313142:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773894.313153:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534779392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773894.313161:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773894.313165:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773894.313166:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773894.313171:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2890 00000100:00080000:3.0:1790773894.313174:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773894.313178:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773894.313182:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773894.313192:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534779520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773894.313201:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773894.313204:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773894.313206:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773894.313210:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2890 00000100:00080000:3.0:1790773894.313213:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773894.313218:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773894.313222:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773894.313232:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534779648/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773894.313247:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773894.313250:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773894.313253:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773894.313257:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2890 00000100:00080000:3.0:1790773894.313260:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773894.313265:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773894.313269:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773894.313279:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534779776/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773894.313335:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534778752:192.168.204.150@tcp:250:kworker.0 00000800:00000100:1.0:1790773894.313394:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773894.313430:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773894/real 1790773894] req@ffff9c4ef73c1c00 x1877759534778752/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773894.313445:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534778752/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773894.313452:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773894.313455:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773894.313458:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534778752:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773894.313463:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534778880:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773894.313478:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773894.313486:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773894/real 1790773894] req@ffff9c4ef73c2300 x1877759534778880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773894.313496:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534778880/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773894.313527:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773894.313532:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773894.313549:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534778880:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773894.313556:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534779008:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773894.313576:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773894.313588:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773894/real 1790773894] req@ffff9c4ef73c0e00 x1877759534779008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773894.313600:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534779008/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773894.313609:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773894.313613:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773894.313616:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534779008:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773894.313622:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534779136:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773894.313640:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773894.313696:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773894/real 1790773894] req@ffff9c4ef73c0a80 x1877759534779136/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773894.313708:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534779136/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773894.313717:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773894.313721:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773894.313725:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534779136:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773894.313731:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534779264:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773894.313750:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773894.313760:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773894/real 1790773894] req@ffff9c4ef73c3800 x1877759534779264/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773894.313772:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534779264/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773894.313780:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773894.313783:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773894.313787:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534779264:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773894.313792:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534779392:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773894.313824:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773894.313834:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773894/real 1790773894] req@ffff9c4ef73c2d80 x1877759534779392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773894.313846:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534779392/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773894.313860:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773894.313864:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773894.313867:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534779392:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773894.313873:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534779520:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773894.313900:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773894.313912:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773894/real 1790773894] req@ffff9c4ef73c2a00 x1877759534779520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773894.313923:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534779520/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773894.313931:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773894.313934:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773894.313937:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534779520:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773894.313943:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534779648:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773894.313961:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773894.313972:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773894/real 1790773894] req@ffff9c4ef73c0700 x1877759534779648/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773894.313983:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534779648/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773894.313993:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773894.313997:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773894.314000:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534779648:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773894.314006:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534779776:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773894.314025:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773894.314034:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773894/real 1790773894] req@ffff9c4ef73c1500 x1877759534779776/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773894.314045:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534779776/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773894.314055:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773894.314058:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773894.314063:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534779776:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773894.376779:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773894.376779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773894.376809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773894.377302:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773894.440755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773894.440781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773894.440913:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773894.568805:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773894.568865:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773894.632778:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773894.696810:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773894.760758:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773894.760767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773894.761402:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773894.824780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773894.888768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773894.888768:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773894.888796:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773895.016773:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773895.016889:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773895.017299:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773895.017387:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773895.080774:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773895.080774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773895.080774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773895.080779:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773895.080808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773895.080813:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773895.208755:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773895.272758:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773895.336793:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773895.400780:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773895.400781:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773895.400846:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773895.400846:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773895.464912:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773895.464978:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773895.465118:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773895.592799:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773895.592799:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773895.656786:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773895.720832:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773895.784778:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773895.784778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773895.784778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773895.848911:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773895.912778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773895.912778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773895.912790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773896.040723:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773896.040723:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773896.040761:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773896.040788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773896.104828:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773896.104828:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773896.104828:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773896.104833:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773896.104871:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773896.104911:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773896.232771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773896.296756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773896.360755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773896.424806:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773896.424806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773896.424806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773896.424888:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773896.488850:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773896.488897:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773896.489002:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773896.617253:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773896.617402:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773896.680799:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773896.744749:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773896.808756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773896.808756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773896.808756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773896.872814:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773896.936767:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773896.936891:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773896.937304:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773897.064824:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773897.064827:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773897.064958:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773897.065672:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773897.128772:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773897.128773:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773897.128774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773897.128813:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773897.128813:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773897.128846:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773897.256772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773897.320784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773897.384821:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773897.448733:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773897.448733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773897.448733:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773897.448742:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773897.512827:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773897.512868:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773897.512896:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773897.640779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773897.640810:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773897.704792:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773897.768753:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773897.832769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773897.832769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773897.832881:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773897.896771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773897.960783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773897.960783:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773897.960784:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773898.088769:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773898.088770:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773898.088772:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773898.088782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773898.152821:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773898.152827:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773898.152846:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773898.152870:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773898.152881:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773898.153685:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773898.280761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773898.344729:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773898.408773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773898.472759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773898.472759:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773898.472865:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773898.472870:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773898.536813:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773898.536859:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773898.536906:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773898.664752:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773898.664785:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773898.728711:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773898.792746:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773898.856728:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773898.856731:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773898.856732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773898.920881:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773898.984803:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773898.984809:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773898.984809:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773899.112785:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773899.112788:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773899.112788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773899.113243:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773899.176806:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773899.176809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773899.176828:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773899.176831:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773899.176874:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773899.176885:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773899.304775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773899.368825:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773899.432856:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773899.432870:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773899.432877:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773899.432893:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2895 00000100:00080000:0.0:1790773899.432911:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773899.432927:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773899.432984:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534779904/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773899.433014:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773899.433018:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773899.433022:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773899.433030:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773899.433038:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2895 00000100:00080000:0.0:1790773899.433044:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773899.433053:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773899.433065:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773899.433082:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534780032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773899.433092:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773899.433096:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773899.433098:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773899.433103:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2895 00000100:00080000:0.0:1790773899.433106:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773899.433111:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773899.433115:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773899.433125:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534780160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773899.433134:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773899.433138:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773899.433140:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773899.433144:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2895 00000100:00080000:0.0:1790773899.433148:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773899.433338:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534779904:192.168.204.150@tcp:250:kworker.0 00000800:00000100:1.0:1790773899.433416:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773899.433468:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773899/real 1790773899] req@ffff9c4ec7a7d180 x1877759534779904/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773899.433487:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534779904/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773899.433497:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773899.433503:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773899.433507:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534779904:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773899.433515:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534780032:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773899.433536:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773899.433561:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773899/real 1790773899] req@ffff9c4ec7a7f480 x1877759534780032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773899.433574:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534780032/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773899.433593:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773899.433596:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773899.433600:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534780032:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773899.433606:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534780160:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773899.433626:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773899.433637:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773899/real 1790773899] req@ffff9c4ec7a7ed80 x1877759534780160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773899.433678:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534780160/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773899.433688:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773899.433691:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773899.433694:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534780160:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773899.434316:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773899.434327:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773899.434348:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534780288/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773899.434374:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773899.434378:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773899.434380:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773899.434385:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2895 00000100:00080000:0.0:1790773899.434392:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773899.434399:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773899.434404:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773899.434414:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534780416/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773899.434424:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773899.434428:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773899.434429:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773899.434434:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2895 00000100:00080000:0.0:1790773899.434438:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773899.434443:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773899.434448:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773899.434457:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534780544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773899.434466:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773899.434470:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773899.434478:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773899.434482:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2895 00000100:00080000:0.0:1790773899.434486:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773899.434491:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773899.434497:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773899.434510:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534780672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773899.434519:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773899.434523:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773899.434525:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773899.434532:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2895 00000100:00080000:0.0:1790773899.434538:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773899.434554:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773899.434557:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534780288:192.168.204.150@tcp:8:kworker.0 00000008:00100000:0.0:1790773899.434561:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773899.434580:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534780800/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773899.434592:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773899.434596:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773899.434598:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773899.434602:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2895 00000100:00080000:0.0:1790773899.434607:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773899.434616:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773899.434623:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773899.434635:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534780928/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773899.434646:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773899.434721:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773899/real 1790773899] req@ffff9c4ec7a7df80 x1877759534780288/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773899.434737:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534780288/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773899.434748:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773899.434753:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773899.434757:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534780288:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773899.434765:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534780416:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773899.434856:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773899.434868:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773899/real 1790773899] req@ffff9c4ec7a7d880 x1877759534780416/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773899.434881:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534780416/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773899.434890:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773899.434894:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773899.434898:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534780416:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773899.434904:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534780544:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773899.434926:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773899.434939:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773899/real 1790773899] req@ffff9c4ec7a7c380 x1877759534780544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773899.434956:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534780544/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773899.434973:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773899.434977:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773899.434980:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534780544:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773899.434987:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534780672:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773899.435005:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773899.435015:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773899/real 1790773899] req@ffff9c4ec7a7ca80 x1877759534780672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773899.435027:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534780672/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773899.435035:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773899.435039:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773899.435043:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534780672:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773899.435067:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534780800:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773899.435082:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773899.435093:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773899/real 1790773899] req@ffff9c4ec7a7fb80 x1877759534780800/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773899.435102:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534780800/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773899.435118:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773899.435122:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773899.435125:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534780800:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773899.435132:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534780928:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773899.435151:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773899.435162:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773899/real 1790773899] req@ffff9c4ec7a7e300 x1877759534780928/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773899.435173:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534780928/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773899.435182:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773899.435186:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773899.435190:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534780928:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773899.496768:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773899.496777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773899.496836:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773899.496866:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773899.560775:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773899.560788:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773899.560825:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773899.688770:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773899.688779:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773899.752804:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773899.816844:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773899.880771:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773899.880771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773899.880899:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773899.945012:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773900.008725:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773900.008728:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773900.008735:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773900.136771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773900.136772:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773900.136970:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773900.137012:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773900.200777:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773900.200777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773900.200783:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773900.200799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773900.200806:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773900.200820:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773900.328764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773900.392775:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773900.456783:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773900.520775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773900.520775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773900.520890:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773900.521605:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773900.584800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773900.584890:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773900.584892:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773900.712790:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773900.712801:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773900.776777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773900.840781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773900.904774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773900.904774:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773900.904892:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773900.968777:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773901.032819:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773901.032820:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773901.032820:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773901.160759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773901.160775:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773901.160775:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773901.160783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773901.224861:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773901.224869:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773901.225029:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773901.225093:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773901.225113:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773901.225713:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773901.352770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773901.416830:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773901.480824:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773901.544739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773901.544744:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773901.544784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773901.544784:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00010000:00010000:0.0:1790773901.608860:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773901.609047:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773901.609571:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773901.736802:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773901.736810:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773901.800856:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773901.864920:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773901.928794:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773901.928794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773901.928917:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773901.992723:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773902.056747:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773902.056747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773902.056747:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773902.184771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773902.185061:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773902.185358:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773902.187463:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773902.248821:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773902.248828:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773902.248832:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773902.248846:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773902.248868:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773902.248868:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773902.376738:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773902.440773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773902.504788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773902.568776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773902.568776:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773902.568783:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773902.568881:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773902.632799:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773902.633364:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773902.633374:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773902.760751:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773902.760802:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773902.824804:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773902.888768:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773902.952767:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773902.952767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773902.952859:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773903.017237:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773903.080796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773903.080798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773903.080799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773903.208752:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773903.208752:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773903.208801:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773903.208859:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773903.272816:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773903.272816:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773903.272829:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773903.272832:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773903.272859:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773903.272884:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773903.400754:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773903.464773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773903.528796:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773903.592771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773903.592771:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773903.592773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773903.592784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773903.656787:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773903.656810:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773903.656931:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773903.784771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773903.784771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773903.848760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773903.912785:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773903.976755:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773903.976755:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773903.976756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773904.040812:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773904.104757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773904.104757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773904.104776:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773904.232797:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773904.232798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773904.232807:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773904.232934:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773904.296805:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773904.296824:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773904.296840:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773904.296861:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773904.296881:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773904.296900:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773904.425118:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773904.488824:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773904.552771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773904.552852:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773904.552860:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773904.552865:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773904.552876:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2900 00000100:00080000:1.0:1790773904.552888:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773904.552907:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773904.552959:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534781056/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773904.552986:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773904.552990:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773904.552992:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773904.552996:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2900 00000100:00080000:1.0:1790773904.553000:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773904.553007:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773904.553019:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773904.553033:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534781056:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773904.553039:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534781184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773904.553053:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773904.553057:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773904.553059:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773904.553066:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2900 00000100:00080000:1.0:1790773904.553071:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773904.553077:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773904.553083:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773904.553104:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534781312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773904.553112:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773904.553115:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773904.553116:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790773904.553122:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773904.553126:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2900 00000100:00080000:1.0:1790773904.553134:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773904.553141:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773904.553147:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773904.553158:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534781440/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773904.553170:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773904.553173:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773904.553176:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773904.553183:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2900 00000100:00100000:2.0:1790773904.553186:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773904/real 1790773904] req@ffff9c4efac4df80 x1877759534781056/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773904.553190:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773904.553200:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773904.553213:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773904.553224:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534781056/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773904.553229:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534781568/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773904.553277:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773904.553280:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773904.553281:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773904.553286:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773904.553296:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2900 00000100:00080000:2.0:1790773904.553297:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773904.553302:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773904.553308:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773904.553313:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773904.553315:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534781056:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773904.553324:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534781696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773904.553332:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773904.553335:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773904.553336:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773904.553342:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2900 00000100:00080000:1.0:1790773904.553344:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773904.553346:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773904.553349:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773904.553357:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534781184:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773904.553359:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534781824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773904.553383:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773904.553385:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773904.553387:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773904.553392:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2900 00000100:00080000:1.0:1790773904.553397:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773904.553402:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773904.553405:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:2.0:1790773904.553412:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773904.553422:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534781952/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773904.553431:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773904.553433:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773904.553438:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773904.553440:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773904/real 1790773904] req@ffff9c4efac4e300 x1877759534781184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773904.553455:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2900 00000100:00080000:1.0:1790773904.553459:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773904.553472:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773904.553478:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773904.553492:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534782080/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773904.553496:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534781184/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773904.553539:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773904.553544:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773904.553549:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534781184:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773904.553557:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534781312:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773904.553581:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773904.553593:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773904/real 1790773904] req@ffff9c4efac4ea00 x1877759534781312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773904.553605:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534781312/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773904.553620:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773904.553624:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773904.553628:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534781312:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773904.553634:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534781440:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773904.553674:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773904.553686:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773904/real 1790773904] req@ffff9c4efac4fb80 x1877759534781440/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773904.553698:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534781440/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773904.553708:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773904.553712:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773904.553716:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534781440:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773904.553722:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534781568:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773904.553739:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773904.553749:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773904/real 1790773904] req@ffff9c4efac4dc00 x1877759534781568/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773904.553761:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534781568/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773904.553770:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773904.553773:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773904.553777:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534781568:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773904.553783:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534781696:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773904.553801:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773904.553812:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773904/real 1790773904] req@ffff9c4efac4e680 x1877759534781696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773904.553823:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534781696/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773904.553837:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773904.553840:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773904.553843:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534781696:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773904.553863:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534781824:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773904.553882:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773904.553893:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773904/real 1790773904] req@ffff9c4efac4f480 x1877759534781824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773904.553905:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534781824/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773904.553913:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773904.553916:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773904.553920:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534781824:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773904.553926:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534781952:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773904.553941:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773904.553951:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773904/real 1790773904] req@ffff9c4efac4ed80 x1877759534781952/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773904.553963:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534781952/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773904.553973:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773904.553976:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773904.553979:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534781952:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773904.553986:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534782080:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773904.554003:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773904.554012:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773904/real 1790773904] req@ffff9c4efac4c700 x1877759534782080/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773904.554024:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534782080/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773904.554033:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773904.554036:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773904.554040:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534782080:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773904.616797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773904.616802:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773904.616806:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773904.616808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773904.680768:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773904.680794:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773904.680931:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773904.808843:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773904.808954:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773904.872821:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773904.936845:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773905.000788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773905.000788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773905.000790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773905.064746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773905.128766:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773905.128766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773905.128782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773905.256765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773905.256770:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773905.256898:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773905.257392:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773905.320843:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773905.320843:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773905.320973:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773905.320991:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773905.321004:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773905.321023:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773905.448750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773905.512782:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773905.576883:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773905.640767:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773905.640767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773905.640767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773905.640767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773905.704765:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773905.704870:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773905.704904:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773905.832774:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773905.832776:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773905.896759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773905.960755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773906.024765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773906.024765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773906.024765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773906.088929:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773906.152771:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773906.152771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773906.152771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773906.280771:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773906.280771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773906.280771:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773906.281399:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773906.344776:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773906.344795:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773906.345182:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773906.345266:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773906.345671:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773906.345862:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773906.472798:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773906.536842:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773906.600786:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773906.664766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773906.664766:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773906.664811:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773906.664877:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00010000:00010000:1.0:1790773906.728820:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773906.729022:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773906.729090:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773906.856786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773906.857514:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773906.920781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773906.984803:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773907.048770:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773907.048770:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773907.048770:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773907.112745:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773907.176819:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773907.176819:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773907.176819:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773907.304758:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773907.304759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773907.304873:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773907.305068:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773907.368815:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773907.368834:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773907.368897:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773907.369769:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773907.369781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773907.369785:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773907.496761:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773907.560971:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773907.624787:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773907.688783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773907.688790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773907.688790:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773907.689467:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773907.752792:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773907.752864:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773907.752969:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773907.880766:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773907.880823:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773907.944744:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773908.008822:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773908.072753:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773908.072901:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773908.072981:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773908.136765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773908.200773:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773908.200843:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773908.201444:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773908.328743:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773908.328743:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773908.328743:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773908.328804:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773908.392775:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773908.392775:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773908.392783:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773908.392918:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773908.392935:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773908.393049:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773908.520777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773908.584993:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773908.648816:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773908.712854:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773908.712904:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773908.712957:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773908.713200:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773908.776874:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773908.777893:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773908.778312:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773908.904786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773908.904786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773908.968780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773909.032792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773909.096804:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773909.097383:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773909.097466:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773909.162685:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773909.224787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773909.224787:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773909.224903:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773909.352774:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773909.352778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773909.352800:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773909.352836:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773909.416726:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773909.416996:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773909.417027:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773909.417045:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773909.417060:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773909.417107:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773909.544786:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773909.608780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:0.0:1790773909.672866:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773909.672876:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773909.672882:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773909.672893:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2905 00000100:00100000:2.0:1790773909.672901:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773909.672904:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773909.672919:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773909.672959:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534782208/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773909.672982:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773909.672986:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773909.672988:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773909.672992:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2905 00000100:00080000:0.0:1790773909.672995:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773909.673001:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773909.673013:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773909.673017:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534782208:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773909.673030:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534782336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773909.673041:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773909.673044:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773909.673046:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773909.673049:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2905 00000100:00080000:0.0:1790773909.673056:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773909.673060:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773909.673064:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773909.673075:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534782464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790773909.673080:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773909.673087:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773909.673090:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773909.673092:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773909.673095:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2905 00000100:00080000:0.0:1790773909.673098:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773909.673110:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773909.673116:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773909.673124:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773909/real 1790773909] req@ffff9c4ec7a7c700 x1877759534782208/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773909.673129:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534782592/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773909.673141:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773909.673144:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773909.673148:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773909.673158:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2905 00000100:00080000:0.0:1790773909.673163:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773909.673168:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773909.673174:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534782208/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773909.673176:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:2.0:1790773909.673190:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1790773909.673193:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534782720/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773909.673198:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773909.673208:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534782208:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773909.673211:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773909.673217:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773909.673220:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773909.673225:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2905 00000100:00080000:0.0:1790773909.673228:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773909.673237:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773909.673237:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534782336:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773909.673245:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773909.673260:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534782848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790773909.673269:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773909.673275:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773909.673280:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773909.673282:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773909.673287:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773909/real 1790773909] req@ffff9c4ec7a7e300 x1877759534782336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773909.673290:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2905 00000100:00080000:0.0:1790773909.673297:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773909.673307:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773909.673311:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534782336/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773909.673315:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773909.673334:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534782976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773909.673335:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773909.673344:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773909.673364:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534782336:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773909.673367:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773909.673372:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773909.673376:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773909.673379:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534782464:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773909.673382:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2905 00000100:00080000:0.0:1790773909.673389:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773909.673395:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773909.673401:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:2.0:1790773909.673406:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773909.673417:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534783104/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773909.673423:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773909/real 1790773909] req@ffff9c4ec7a7fb80 x1877759534782464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773909.673440:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773909.673445:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1790773909.673447:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534782464/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773909.673448:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773909.673457:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2905 00000100:00080000:0.0:1790773909.673465:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773909.673466:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773909.673472:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773909.673474:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773909.673481:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534782464:192.168.204.150@tcp:38:kworker.0 00000008:00100000:0.0:1790773909.673482:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773909.673490:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534782592:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773909.673497:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534783232/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790773909.673511:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773909.673524:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773909/real 1790773909] req@ffff9c4ec7a7ca80 x1877759534782592/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773909.673536:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534782592/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773909.673545:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773909.673549:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773909.673552:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534782592:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773909.673558:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534782720:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773909.673572:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773909.673582:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773909/real 1790773909] req@ffff9c4ec7a7c380 x1877759534782720/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773909.673609:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534782720/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773909.673619:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773909.673622:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773909.673625:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534782720:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773909.673641:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534782848:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773909.673686:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773909.673697:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773909/real 1790773909] req@ffff9c4ec7a7d880 x1877759534782848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773909.673709:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534782848/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773909.673725:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773909.673728:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773909.673731:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534782848:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773909.673737:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534782976:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773909.673752:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773909.673771:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773909/real 1790773909] req@ffff9c4ec7a7df80 x1877759534782976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773909.673784:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534782976/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773909.673793:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773909.673796:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773909.673799:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534782976:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773909.673805:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534783104:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773909.673822:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773909.673831:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773909/real 1790773909] req@ffff9c4ec7a7ed80 x1877759534783104/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773909.673841:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534783104/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773909.673853:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773909.673856:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773909.673859:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534783104:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773909.673865:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534783232:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773909.673883:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773909.673892:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773909/real 1790773909] req@ffff9c4ec7a7f480 x1877759534783232/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773909.673904:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534783232/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773909.673914:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773909.673916:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773909.673920:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534783232:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773909.736746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773909.736746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773909.736780:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773909.736938:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773909.800764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773909.800880:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773909.801039:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773909.928785:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773909.928798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773909.992827:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773910.057007:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773910.120759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773910.120761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773910.121004:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773910.184864:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773910.248743:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773910.248769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773910.248812:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773910.376781:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773910.376793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773910.376793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773910.376811:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773910.440841:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773910.440842:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773910.440842:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773910.440859:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773910.440916:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773910.440924:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773910.568834:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773910.632809:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773910.696792:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773910.760767:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773910.760767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773910.760767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773910.760767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773910.824773:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773910.824837:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773910.824885:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773910.952769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773910.952909:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773911.016937:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773911.080802:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773911.144821:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773911.144865:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773911.144981:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773911.209729:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773911.272805:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773911.272805:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773911.272808:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773911.400799:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773911.400799:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773911.400810:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773911.401306:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773911.464794:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773911.464794:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773911.464850:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773911.464868:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773911.464952:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773911.465045:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773911.593770:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773911.656958:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773911.720771:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773911.784764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773911.784764:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773911.784764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773911.784764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773911.848782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773911.848960:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773911.849009:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773911.976775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773911.976775:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773912.040851:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773912.104801:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773912.168760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773912.168760:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773912.168760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773912.232868:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773912.296729:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773912.296771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773912.297365:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773912.424826:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773912.424827:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773912.424827:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773912.424827:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773912.488779:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773912.488783:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773912.488808:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773912.489192:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773912.489208:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773912.489369:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773912.616782:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773912.680874:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773912.744798:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773912.808810:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773912.808810:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773912.809028:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773912.809275:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773912.872790:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773912.873297:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773912.873549:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773913.000818:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773913.001215:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773913.066110:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773913.128772:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773913.192776:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773913.192776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773913.192827:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773913.256896:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773913.320761:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773913.320761:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773913.321193:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773913.448719:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773913.448749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773913.448760:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773913.448843:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773913.512735:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773913.512738:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773913.512758:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773913.512859:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773913.512898:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773913.513260:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773913.640775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773913.705128:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773913.768780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773913.832756:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773913.832784:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773913.832789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773913.832809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773913.896734:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773913.896768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773913.896775:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773914.024756:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773914.024759:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773914.088757:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773914.152755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773914.216749:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773914.216758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773914.216803:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773914.280779:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773914.344803:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773914.344885:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773914.344885:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773914.472801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773914.472816:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773914.472822:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773914.472954:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773914.536807:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773914.536807:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773914.536835:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773914.536847:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773914.536936:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773914.536987:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773914.664747:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773914.728762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773914.792782:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773914.792792:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773914.792798:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773914.792802:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773914.792811:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2910 00000100:00080000:0.0:1790773914.792821:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773914.792834:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773914.792875:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534783360/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773914.792901:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773914.792904:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773914.792906:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773914.792909:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2910 00000100:00080000:0.0:1790773914.792913:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773914.792918:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773914.792926:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773914.792937:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534783360:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773914.792941:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534783488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773914.792960:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773914.792964:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773914.792966:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773914.792972:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2910 00000100:00080000:0.0:1790773914.792976:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773914.792982:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773914.792987:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773914.792999:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534783616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773914.793009:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773914.793012:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773914.793014:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773914.793018:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2910 00000800:00000100:2.0:1790773914.793023:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773914.793024:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773914.793032:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773914.793039:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773914.793051:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534783744/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773914.793062:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773914.793066:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773914.793070:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773914.793071:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773914/real 1790773914] req@ffff9c4ec7a7d180 x1877759534783360/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773914.793078:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2910 00000100:00080000:0.0:1790773914.793084:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773914.793093:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773914.793097:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773914.793109:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534783872/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773914.793117:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534783360/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773914.793123:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773914.793131:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773914.793135:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773914.793140:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773914.793144:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2910 00000100:00080000:0.0:1790773914.793149:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773914.793152:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773914.793159:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534783360:192.168.204.150@tcp:250:kworker.0 00000100:00080000:0.0:1790773914.793164:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773914.793171:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773914.793181:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534784000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773914.793186:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534783488:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773914.793192:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773914.793196:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773914.793198:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773914.793201:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2910 00000100:00080000:0.0:1790773914.793205:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773914.793209:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773914.793213:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790773914.793220:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:0.0:1790773914.793223:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534784128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773914.793234:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773914.793237:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773914.793238:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773914.793242:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2910 00000100:00080000:0.0:1790773914.793244:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773914.793266:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773914.793266:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773914/real 1790773914] req@ffff9c4ec7a7f480 x1877759534783488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773914.793274:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773914.793289:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534783488/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773914.793292:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534784256/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773914.793311:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773914.793315:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773914.793317:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773914.793322:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2910 00000100:00080000:0.0:1790773914.793326:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773914.793328:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773914.793331:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773914.793336:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790773914.793337:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773914.793343:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534783488:192.168.204.150@tcp:38:kworker.0 00000100:00100000:0.0:1790773914.793353:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534784384/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773914.793360:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534783616:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773914.793390:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773914.793403:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773914/real 1790773914] req@ffff9c4ec7a7ed80 x1877759534783616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773914.793412:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534783616/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773914.793417:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773914.793420:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773914.793422:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534783616:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773914.793427:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534783744:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773914.793448:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773914.793460:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773914/real 1790773914] req@ffff9c4ec7a7df80 x1877759534783744/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773914.793471:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534783744/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773914.793480:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773914.793496:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773914.793501:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534783744:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773914.793507:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534783872:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773914.793528:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773914.793539:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773914/real 1790773914] req@ffff9c4ec7a7d880 x1877759534783872/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773914.793550:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534783872/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773914.793559:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773914.793563:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773914.793574:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534783872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773914.793595:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534784000:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773914.793616:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773914.793627:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773914/real 1790773914] req@ffff9c4ec7a7c380 x1877759534784000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773914.793638:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534784000/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773914.793673:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773914.793677:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773914.793681:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534784000:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773914.793687:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534784128:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773914.793706:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773914.793715:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773914/real 1790773914] req@ffff9c4ec7a7ca80 x1877759534784128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773914.793723:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534784128/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773914.793728:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773914.793730:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773914.793732:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534784128:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773914.793736:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534784256:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773914.793749:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773914.793758:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773914/real 1790773914] req@ffff9c4ec7a7fb80 x1877759534784256/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773914.793768:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534784256/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773914.793784:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773914.793788:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773914.793791:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534784256:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773914.793797:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534784384:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773914.793815:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773914.793826:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773914/real 1790773914] req@ffff9c4ec7a7e300 x1877759534784384/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773914.793837:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534784384/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773914.793846:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773914.793849:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773914.793853:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534784384:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773914.856773:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773914.856774:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773914.856785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773914.856932:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773914.920848:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773914.921322:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773914.922523:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773915.048881:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773915.048917:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773915.112824:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773915.176760:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773915.240873:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773915.240873:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773915.241053:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773915.304929:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773915.368779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773915.368954:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773915.369589:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773915.496888:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773915.496889:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773915.496890:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773915.496903:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773915.560795:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773915.560795:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773915.560795:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773915.560834:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773915.560855:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773915.560871:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773915.688921:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773915.752789:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773915.816981:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773915.880771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773915.880787:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773915.880869:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773915.881355:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773915.944905:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773915.944911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773915.944966:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773916.072839:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773916.073338:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773916.137060:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773916.201695:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773916.264758:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773916.264758:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773916.264992:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773916.328809:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773916.392797:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773916.392857:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773916.393047:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773916.520810:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773916.520812:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773916.520880:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773916.521072:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773916.584793:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773916.584793:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773916.585790:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773916.586084:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773916.587100:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773916.587101:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773916.712766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773916.776801:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773916.840820:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773916.904817:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773916.904817:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773916.904828:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773916.904830:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773916.968792:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773916.969741:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773916.971438:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773917.096770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773917.097105:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773917.160766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773917.224794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773917.288799:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773917.288799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773917.288799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773917.352790:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773917.416859:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773917.417168:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773917.417409:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773917.544768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773917.544768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773917.544796:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773917.545356:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773917.608773:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773917.608789:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773917.608826:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773917.608885:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773917.608902:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773917.608920:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773917.736752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773917.800913:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773917.864785:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773917.928787:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773917.928788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773917.929116:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773917.929137:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773917.992831:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773917.993034:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773917.993725:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773918.120963:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773918.121056:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773918.185237:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773918.249295:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773918.312789:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773918.312792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773918.312888:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773918.376765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773918.440775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773918.440775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773918.440882:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773918.568746:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773918.568754:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773918.568757:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773918.568824:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773918.632816:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773918.632816:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773918.632954:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773918.633020:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773918.633048:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773918.633050:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773918.760775:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773918.824818:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773918.888815:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773918.952769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773918.952769:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773918.952845:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773918.952848:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773919.016803:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773919.016811:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773919.016816:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773919.144757:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773919.144758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773919.208763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773919.272780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773919.336787:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773919.336799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773919.337251:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773919.400746:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773919.464775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773919.464780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773919.464802:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773919.592786:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773919.592789:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773919.593004:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773919.593075:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773919.656802:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773919.656811:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773919.656815:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773919.656838:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773919.656848:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773919.656866:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773919.784792:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773919.848769:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773919.912789:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773919.912803:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773919.912810:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773919.912824:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2915 00000100:00100000:2.0:1790773919.912834:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773919.912839:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773919.912854:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773919.912902:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534784512/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773919.912926:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773919.912929:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773919.912931:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773919.912934:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2915 00000100:00080000:1.0:1790773919.912937:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773919.912940:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773919.912951:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773919.912961:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534784640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773919.912969:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773919.912972:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773919.912973:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773919.912978:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2915 00000100:00080000:1.0:1790773919.912981:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773919.912987:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534784512:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773919.912988:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773919.912996:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773919.913006:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534784768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773919.913019:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773919.913023:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773919.913024:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773919.913029:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2915 00000100:00080000:1.0:1790773919.913032:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773919.913036:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773919.913041:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773919.913051:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534784896/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773919.913058:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773919.913061:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773919.913063:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773919.913067:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2915 00000100:00080000:1.0:1790773919.913070:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773919.913073:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773919.913077:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773919.913090:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534785024/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773919.913100:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773919.913106:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534784640:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773919.913107:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773919.913111:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773919.913118:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2915 00000100:00080000:1.0:1790773919.913126:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773919.913135:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773919.913140:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773919.913165:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534784768:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773919.913166:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534785152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773919.913188:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773919.913191:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773919.913194:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773919.913198:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2915 00000100:00080000:1.0:1790773919.913204:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773919.913209:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773919.913213:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773919.913219:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534784896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773919.913231:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534785280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773919.913239:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773919.913242:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773919.913243:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773919.913247:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2915 00000100:00080000:1.0:1790773919.913250:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773919.913253:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773919.913257:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773919.913264:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534785408/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773919.913270:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773919.913272:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773919.913273:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773919.913276:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2915 00000100:00080000:1.0:1790773919.913278:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773919.913280:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773919.913283:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773919.913289:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534785536/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773919.913330:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534785024:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773919.913363:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534785152:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773919.913383:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534785280:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773919.913401:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534785408:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773919.913430:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534785536:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773919.976760:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773919.976792:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773919.976792:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773919.976801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773920.040850:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773920.040851:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773920.040855:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773920.168768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773920.168776:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773920.232790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773920.296796:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773920.360757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773920.360757:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773920.360811:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773920.424765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773920.488762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773920.488767:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773920.488790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773920.616782:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773920.616797:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773920.616877:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773920.617401:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773920.680770:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773920.680770:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773920.680786:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773920.680887:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773920.680913:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773920.680917:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773920.808752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773920.872816:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773920.937027:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773921.000729:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773921.000730:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773921.000739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773921.000831:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773921.064827:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773921.064827:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773921.064917:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773921.192829:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773921.192832:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773921.256793:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773921.321001:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773921.384772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773921.384805:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773921.384811:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773921.448760:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773921.512799:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773921.512799:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773921.513191:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773921.640787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773921.640787:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773921.640793:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773921.640827:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773921.704755:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773921.704757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773921.704766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773921.704772:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773921.704792:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773921.704796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773921.832803:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773921.896801:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773921.960799:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773922.024767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773922.024767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773922.024775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773922.024799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773922.088781:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773922.088785:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773922.088830:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773922.216803:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773922.216875:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773922.280774:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773922.344781:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773922.408816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773922.408824:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773922.408869:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773922.472819:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773922.536791:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773922.536791:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773922.536813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773922.664764:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773922.664764:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773922.664880:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773922.664980:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773922.728757:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773922.728757:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773922.728759:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773922.728780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773922.728798:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773922.728804:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773922.856809:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773922.920827:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773922.984801:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:3.0:1790773922.984917:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:3.0:1790773922.984957:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1790773922.984972:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773922.985021:0:2385:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:3.0:1790773922.985044:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790773922.985051:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773919/real 1790773922] req@ffff9c4efac4e300 x1877759534785536/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1790773922.985058:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773922.985072:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790773922.985084:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534785536/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1790773922.985103:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773922.985119:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:2.0:1790773922.985121:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773922.985129:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000800:00000100:3.0:1790773922.985131:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790773922.985137:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534785536:192.168.204.150@tcp:8:kworker.0 00000800:00000100:3.0:1790773922.985143:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:2.0:1790773922.985149:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773919/real 1790773922] req@ffff9c4efac4c700 x1877759534784640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773922.985167:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534784640/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773922.985183:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773922.985188:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773922.985192:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534784640:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773922.985199:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773919/real 1790773922] req@ffff9c4efac4ed80 x1877759534784768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773922.985212:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534784768/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773922.985221:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773922.985225:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773922.985228:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534784768:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773922.985241:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773919/real 1790773922] req@ffff9c4efac4f480 x1877759534784896/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773922.985254:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534784896/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773922.985263:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773922.985266:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773922.985270:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534784896:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773922.985277:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773919/real 1790773922] req@ffff9c4efac4f800 x1877759534784512/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773922.985288:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534784512/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773922.985295:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773922.985299:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773922.985303:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534784512:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773922.985333:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773919/real 1790773922] req@ffff9c4efac4e680 x1877759534785024/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773922.985346:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534785024/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773922.985360:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773922.985365:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773922.985368:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534785024:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773922.985374:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773919/real 1790773922] req@ffff9c4efac4dc00 x1877759534785152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773922.985383:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534785152/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773922.985394:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773922.985397:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773922.985399:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534785152:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773922.985403:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773919/real 1790773922] req@ffff9c4efac4fb80 x1877759534785280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773922.985413:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534785280/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773922.985421:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773922.985424:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773922.985428:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534785280:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773922.985434:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773919/real 1790773922] req@ffff9c4efac4ea00 x1877759534785408/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773922.985446:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534785408/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773922.985456:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773922.985460:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773922.985463:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534785408:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773923.048756:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773923.048757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773923.048757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773923.048758:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00010000:00010000:1.0:1790773923.112813:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773923.112956:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773923.113082:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773923.240810:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773923.240810:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773923.304886:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773923.368889:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773923.432849:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773923.432849:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773923.433218:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773923.496784:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773923.560777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773923.560778:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773923.560808:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773923.688811:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773923.688811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773923.688814:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773923.688978:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773923.752753:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773923.752816:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773923.752864:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773923.752967:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773923.752968:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773923.753008:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773923.880766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773923.944806:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773924.008813:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773924.072813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773924.072814:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773924.072831:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773924.072843:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773924.136755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773924.136789:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773924.136793:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773924.264780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773924.264849:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773924.328813:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773924.392823:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773924.456749:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773924.456816:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773924.456821:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773924.520780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773924.584783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773924.584783:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773924.585173:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773924.712808:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773924.712808:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773924.712828:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773924.712843:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773924.776770:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773924.776801:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773924.776809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773924.776845:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773924.776849:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773924.776879:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773924.904781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773924.968754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773925.032836:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773925.032847:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773925.032859:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773925.032864:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773925.032876:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2920 00000100:00080000:2.0:1790773925.032887:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773925.032899:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773925.032936:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534785664/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773925.032964:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773925.032967:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773925.032969:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773925.032973:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2920 00000100:00080000:2.0:1790773925.032977:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773925.032982:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773925.032991:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773925.033005:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534785792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773925.033016:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534785664:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773925.033018:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773925.033026:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773925.033029:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773925.033033:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2920 00000100:00080000:2.0:1790773925.033037:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773925.033043:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773925.033058:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773925.033074:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534785920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773925.033084:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773925.033088:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773925.033090:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773925.033094:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2920 00000100:00080000:2.0:1790773925.033099:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773925.033103:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773925.033107:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773925.033113:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773925.033127:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534786048/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773925.033136:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773925.033139:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773925.033141:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773925.033151:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773925/real 1790773925] req@ffff9c4efbcc0380 x1877759534785664/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773925.033153:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2920 00000100:00080000:2.0:1790773925.033159:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773925.033167:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773925.033173:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773925.033180:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534785664/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773925.033186:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534786176/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773925.033199:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773925.033202:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773925.033207:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773925.033208:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773925.033212:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773925.033214:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534785664:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773925.033222:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2920 00000100:00080000:2.0:1790773925.033227:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773925.033233:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773925.033238:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773925.033247:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534785792:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773925.033253:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534786304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773925.033266:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773925.033270:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773925.033272:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773925.033277:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773925.033291:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2920 00000100:00100000:1.0:1790773925.033294:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773925/real 1790773925] req@ffff9c4efbcc2300 x1877759534785792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773925.033297:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773925.033312:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773925.033315:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534785792/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773925.033319:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773925.033337:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534786432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773925.033341:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773925.033349:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773925.033352:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773925.033356:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534785792:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773925.033358:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773925.033362:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773925.033369:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534785920:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773925.033369:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2920 00000100:00080000:2.0:1790773925.033376:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773925.033383:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773925.033389:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:1.0:1790773925.033400:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773925.033403:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534786560/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773925.033416:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773925/real 1790773925] req@ffff9c4efbcc1f80 x1877759534785920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773925.033424:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773925.033430:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1790773925.033433:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534785920/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773925.033435:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773925.033444:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2920 00000100:00080000:1.0:1790773925.033448:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773925.033451:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773925.033455:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773925.033460:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773925.033463:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534785920:192.168.204.150@tcp:38:kworker.0 00000008:00100000:2.0:1790773925.033468:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773925.033473:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534786048:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773925.033484:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534786688/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773925.033507:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773925.033521:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773925/real 1790773925] req@ffff9c4efbcc1c00 x1877759534786048/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773925.033534:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534786048/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773925.033544:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773925.033547:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773925.033551:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534786048:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773925.033557:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534786176:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773925.033576:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773925.033588:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773925/real 1790773925] req@ffff9c4efbcc3100 x1877759534786176/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773925.033610:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534786176/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773925.033620:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773925.033623:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773925.033627:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534786176:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773925.033648:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534786304:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773925.033697:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773925.033709:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773925/real 1790773925] req@ffff9c4efbcc2680 x1877759534786304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773925.033721:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534786304/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773925.033738:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773925.033742:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773925.033746:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534786304:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773925.033753:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534786432:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773925.033773:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773925.033784:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773925/real 1790773925] req@ffff9c4efbcc1880 x1877759534786432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773925.033813:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534786432/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773925.033827:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773925.033831:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773925.033834:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534786432:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773925.033840:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534786560:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773925.033860:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773925.033871:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773925/real 1790773925] req@ffff9c4efbcc1500 x1877759534786560/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773925.033881:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534786560/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773925.033892:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773925.033895:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773925.033898:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534786560:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773925.033904:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534786688:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773925.033921:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773925.033931:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773925/real 1790773925] req@ffff9c4efbcc0700 x1877759534786688/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773925.033942:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534786688/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773925.033951:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773925.033954:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773925.033958:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534786688:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773925.096845:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773925.096854:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773925.097033:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773925.097679:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773925.160772:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773925.160825:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773925.160869:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773925.288770:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773925.288786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773925.352815:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773925.416949:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773925.480805:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773925.481020:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773925.482504:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773925.544772:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773925.608793:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773925.608793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773925.608823:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773925.736782:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773925.736793:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773925.736854:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773925.737341:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773925.800761:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773925.800794:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773925.800805:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773925.800898:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773925.800919:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773925.801374:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773925.928759:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773925.992837:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773926.056999:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773926.120753:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773926.120772:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773926.120773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773926.120792:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773926.184807:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773926.184860:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773926.184915:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773926.312807:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773926.313208:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773926.376766:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773926.440779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773926.504808:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773926.504834:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773926.504875:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773926.568743:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773926.632795:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773926.632795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773926.632859:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773926.760817:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773926.760817:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773926.760844:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773926.761037:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773926.824786:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773926.824802:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773926.824965:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773926.825097:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773926.825267:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773926.825314:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773926.952869:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773927.016820:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773927.080821:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773927.144785:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773927.144792:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773927.144805:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773927.144813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00010000:00010000:1.0:1790773927.208838:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773927.208880:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773927.208907:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773927.337490:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773927.338463:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773927.400813:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773927.464798:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773927.529013:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773927.529109:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773927.529120:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773927.592827:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773927.656818:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773927.656829:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773927.656882:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773927.784787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773927.784818:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773927.784848:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773927.784849:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773927.848743:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773927.848749:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773927.848839:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773927.849028:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773927.849038:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773927.852713:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773927.976847:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773928.040783:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773928.104780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773928.168818:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773928.168818:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773928.168818:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773928.168885:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773928.232807:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773928.232834:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773928.232911:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773928.360913:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773928.360936:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773928.424759:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773928.488910:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773928.552815:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773928.552818:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773928.552841:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773928.616784:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773928.680777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773928.680777:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773928.680807:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773928.808828:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773928.808830:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773928.809005:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773928.809046:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773928.872815:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773928.872908:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773928.872912:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773928.872922:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773928.872952:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773928.872995:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773929.000800:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773929.064790:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773929.128815:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773929.192781:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773929.193244:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773929.193714:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773929.193727:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773929.256762:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773929.257034:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773929.257089:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773929.384782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773929.384790:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773929.448797:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773929.512820:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773929.576809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773929.576835:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773929.577230:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773929.640808:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773929.704794:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773929.704812:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773929.704841:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773929.832835:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773929.832835:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773929.832836:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773929.832836:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773929.897188:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773929.897486:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773929.897706:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773929.897907:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773929.898065:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773929.898233:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773930.024841:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773930.088761:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773930.152739:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773930.152753:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773930.152760:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773930.152764:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773930.152775:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2926 00000100:00080000:2.0:1790773930.152784:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773930.152796:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773930.152830:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534786816/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773930.152855:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773930.152860:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773930.152862:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773930.152867:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2926 00000100:00080000:2.0:1790773930.152871:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773930.152877:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773930.152886:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773930.152890:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534786816:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773930.152902:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534786944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773930.152914:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773930.152918:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773930.152920:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773930.152932:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2926 00000100:00080000:2.0:1790773930.152936:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773930.152943:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773930.152943:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773930.152962:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773930.152975:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534787072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773930.152990:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773930.152994:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1790773930.152997:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773930/real 1790773930] req@ffff9c4efbcc0a80 x1877759534786816/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773930.152997:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773930.153007:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2926 00000100:00080000:2.0:1790773930.153013:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773930.153016:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534786816/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773930.153021:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773930.153029:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1790773930.153030:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00080000:1.0:1790773930.153034:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773930.153038:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534786816:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773930.153047:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534787200/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773930.153058:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534786944:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773930.153059:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773930.153065:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773930.153068:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773930.153073:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2926 00000800:00000100:1.0:1790773930.153079:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773930.153089:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773930/real 1790773930] req@ffff9c4efbcc2a00 x1877759534786944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773930.153090:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773930.153097:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773930.153105:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773930.153110:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534786944/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773930.153119:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534787328/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773930.153130:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773930.153131:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773930.153136:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773930.153140:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773930.153145:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2926 00000100:00080000:1.0:1790773930.153148:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:2.0:1790773930.153149:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773930.153158:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534786944:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773930.153159:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773930.153167:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773930.153172:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534787072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773930.153181:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534787456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773930.153191:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773930.153195:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773930.153197:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773930.153201:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2926 00000100:00080000:2.0:1790773930.153205:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773930.153209:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773930.153213:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773930.153218:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773930.153228:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773930/real 1790773930] req@ffff9c4efbcc0700 x1877759534787072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773930.153230:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534787584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773930.153253:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773930.153262:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773930.153264:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773930.153270:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534787072/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773930.153270:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2926 00000100:00080000:2.0:1790773930.153274:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773930.153282:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773930.153286:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773930.153291:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000008:00100000:2.0:1790773930.153298:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773930.153299:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534787072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773930.153313:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534787712/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773930.153324:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534787200:192.168.204.150@tcp:8:kworker.0 00000100:00080000:2.0:1790773930.153324:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773930.153332:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773930.153334:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773930.153340:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2926 00000100:00080000:2.0:1790773930.153344:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773930.153349:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773930.153354:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000800:00000100:1.0:1790773930.153357:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773930.153367:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534787840/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773930.153371:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773930/real 1790773930] req@ffff9c4efbcc1500 x1877759534787200/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773930.153392:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534787200/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773930.153404:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773930.153407:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773930.153412:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534787200:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773930.153419:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534787328:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773930.153437:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773930.153448:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773930/real 1790773930] req@ffff9c4efbcc1880 x1877759534787328/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773930.153459:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534787328/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773930.153467:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773930.153469:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773930.153471:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534787328:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773930.153475:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534787456:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773930.153489:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773930.153496:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773930/real 1790773930] req@ffff9c4efbcc2680 x1877759534787456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773930.153503:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534787456/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773930.153514:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773930.153515:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773930.153518:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534787456:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773930.153522:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534787584:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773930.153533:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773930.153541:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773930/real 1790773930] req@ffff9c4efbcc3100 x1877759534787584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773930.153552:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534787584/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773930.153560:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773930.153563:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773930.153572:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534787584:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773930.153589:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534787712:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773930.153609:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773930.153619:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773930/real 1790773930] req@ffff9c4efbcc1c00 x1877759534787712/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773930.153631:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534787712/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773930.153641:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773930.153644:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773930.153647:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534787712:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773930.153676:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534787840:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773930.153691:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773930.153698:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773930/real 1790773930] req@ffff9c4efbcc1f80 x1877759534787840/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773930.153716:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534787840/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773930.153726:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773930.153729:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773930.153732:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534787840:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773930.216806:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773930.216809:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773930.216812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773930.216883:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773930.280836:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773930.280960:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773930.281022:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773930.408767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773930.408770:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773930.472784:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773930.536864:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773930.600796:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773930.600796:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773930.600809:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773930.664769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773930.728762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773930.728763:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773930.728833:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773930.856806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773930.856824:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773930.856824:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773930.857034:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773930.920724:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773930.920726:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773930.920735:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773930.920748:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773930.920762:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773930.920778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773931.048819:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773931.112818:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773931.176790:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773931.240762:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773931.240762:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773931.240780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773931.240893:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773931.304732:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773931.304768:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773931.304775:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773931.432841:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773931.432890:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773931.496771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773931.560767:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773931.624814:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773931.624815:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773931.624815:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773931.688769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773931.752751:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773931.752753:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773931.752813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773931.880811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773931.880811:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773931.880820:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773931.880860:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773931.944808:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773931.944833:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773931.944835:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773931.944836:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773931.944866:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773931.944866:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773932.072804:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773932.136862:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773932.200797:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773932.264833:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773932.264833:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773932.264833:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773932.264871:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773932.328821:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773932.328821:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773932.328998:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773932.456752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773932.456768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773932.520782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773932.584772:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773932.648767:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773932.648767:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773932.648775:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773932.712775:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773932.776757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773932.776776:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773932.776783:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773932.904757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773932.904765:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773932.904785:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773932.905020:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773932.968798:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773932.968813:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773932.968845:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773932.968975:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773932.968978:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773932.969041:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773933.096785:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773933.162054:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773933.224773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773933.288844:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773933.288845:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773933.288845:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773933.288985:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773933.352766:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773933.352779:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773933.352910:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773933.480781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773933.480841:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773933.544816:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773933.608724:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773933.672709:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773933.672715:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773933.673153:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773933.736770:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773933.800757:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773933.800766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773933.800868:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773933.928793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773933.928793:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773933.928793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773933.928845:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773933.992827:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773933.992830:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773933.992906:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773933.992941:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773933.992980:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773933.993025:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773934.120936:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773934.184967:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773934.248904:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773934.312793:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773934.312798:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773934.312818:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773934.312836:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773934.376813:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773934.376844:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773934.376888:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773934.504769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773934.504769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773934.568813:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773934.632846:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773934.696821:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773934.696824:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773934.696834:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773934.760763:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773934.824778:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773934.824785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773934.824785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773934.952741:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773934.952815:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773934.952885:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773934.952928:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773935.016816:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773935.016816:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773935.016816:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773935.016818:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773935.016878:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773935.016892:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773935.144816:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773935.208843:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773935.272780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773935.272799:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773935.272807:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773935.272811:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773935.272826:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2931 00000100:00080000:0.0:1790773935.272839:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773935.272868:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773935.272913:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c700 x1877759534787968/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773935.272960:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773935.272964:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773935.272967:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773935.272970:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2931 00000100:00080000:0.0:1790773935.272973:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773935.272978:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773935.272988:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773935.273008:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534787968:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773935.273011:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534788096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773935.273033:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773935.273036:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773935.273037:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773935.273041:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2931 00000100:00080000:0.0:1790773935.273044:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773935.273048:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773935.273051:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773935.273060:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534788224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773935.273071:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773935.273076:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773935.273077:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773935.273082:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2931 00000100:00080000:0.0:1790773935.273094:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773935.273101:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773935.273106:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773935.273114:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773935.273129:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534788352/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773935.273137:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773935.273141:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773935.273143:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773935.273147:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2931 00000100:00080000:0.0:1790773935.273150:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773935.273154:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773935/real 1790773935] req@ffff9c4ec7a7c700 x1877759534787968/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773935.273159:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773935.273167:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773935.273181:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534788480/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773935.273190:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773935.273196:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1790773935.273197:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c700 x1877759534787968/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773935.273200:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773935.273211:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2931 00000100:00080000:0.0:1790773935.273220:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773935.273223:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1790773935.273229:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773935.273236:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000008:00100000:0.0:1790773935.273239:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773935.273247:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534787968:192.168.204.150@tcp:250:kworker.0 00000100:00100000:0.0:1790773935.273260:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534788608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773935.273276:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773935.273281:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773935.273283:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773935.273288:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534788096:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773935.273290:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2931 00000100:00080000:0.0:1790773935.273309:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1790773935.273325:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773935.273330:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773935.273338:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773935.273345:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773935/real 1790773935] req@ffff9c4ec7a7e300 x1877759534788096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773935.273357:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534788736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773935.273367:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534788096/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1790773935.273373:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773935.273385:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773935.273390:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773935.273407:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2931 00000100:00080000:0.0:1790773935.273416:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773935.273424:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773935.273429:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1790773935.273430:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:1.0:1790773935.273441:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:0.0:1790773935.273455:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534788864/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773935.273455:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534788096:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773935.273477:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534788224:192.168.204.150@tcp:38:kworker.0 00000100:00080000:0.0:1790773935.273500:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773935.273504:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773935.273507:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773935.273515:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2931 00000800:00000100:1.0:1790773935.273519:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:0.0:1790773935.273524:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773935.273545:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773935.273551:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773935/real 1790773935] req@ffff9c4ec7a7fb80 x1877759534788224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1790773935.273555:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773935.273577:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534788992/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773935.273581:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534788224/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773935.273622:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773935.273629:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773935.273635:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534788224:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773935.273643:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534788352:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773935.273696:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773935.273709:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773935/real 1790773935] req@ffff9c4ec7a7ca80 x1877759534788352/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773935.273721:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534788352/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773935.273730:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773935.273733:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773935.273737:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534788352:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773935.273744:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534788480:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773935.273764:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773935.273776:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773935/real 1790773935] req@ffff9c4ec7a7c380 x1877759534788480/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773935.273788:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534788480/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773935.273796:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773935.273800:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773935.273803:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534788480:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773935.273821:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534788608:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773935.273843:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773935.273873:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773935/real 1790773935] req@ffff9c4ec7a7d880 x1877759534788608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773935.273885:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534788608/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773935.273901:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773935.273905:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773935.273908:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534788608:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773935.273914:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534788736:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773935.273936:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773935.273948:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773935/real 1790773935] req@ffff9c4ec7a7df80 x1877759534788736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773935.273960:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534788736/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773935.273968:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773935.273972:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773935.273975:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534788736:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773935.273988:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534788864:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773935.274007:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773935.274018:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773935/real 1790773935] req@ffff9c4ec7a7ed80 x1877759534788864/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773935.274028:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534788864/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773935.274038:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773935.274041:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773935.274044:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534788864:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773935.274051:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534788992:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773935.274070:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773935.274080:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773935/real 1790773935] req@ffff9c4ec7a7f480 x1877759534788992/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773935.274091:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534788992/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773935.274101:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773935.274104:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773935.274108:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534788992:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773935.336784:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773935.336784:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773935.336784:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773935.336857:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773935.400769:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773935.400769:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773935.400861:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773935.528790:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773935.529504:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773935.592747:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773935.656806:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773935.720760:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773935.720760:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773935.720762:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773935.784761:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773935.848772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773935.848772:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773935.848937:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773935.976761:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773935.976762:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773935.976773:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773935.976801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773936.040763:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773936.040763:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773936.040770:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773936.040950:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773936.040954:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773936.040986:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773936.168800:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773936.232754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773936.296765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773936.360746:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773936.360746:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773936.360746:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773936.360918:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773936.424753:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773936.424774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773936.424800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773936.552771:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773936.552848:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773936.616882:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773936.680728:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773936.744759:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773936.744759:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773936.744791:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773936.808759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773936.872779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773936.872781:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773936.872795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773937.000763:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773937.000763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773937.000786:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773937.000899:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773937.064760:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773937.064761:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773937.064771:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773937.064792:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773937.064860:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773937.064865:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773937.192764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773937.256753:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773937.320797:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773937.384782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773937.384782:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773937.384801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773937.384848:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773937.448833:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773937.448914:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773937.448931:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773937.576789:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773937.576818:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773937.640771:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773937.704756:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773937.768774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773937.768786:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773937.768786:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773937.832766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773937.896782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773937.896811:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773937.896842:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773938.024789:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773938.024789:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773938.024791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773938.024814:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773938.088771:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773938.088771:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773938.088771:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773938.088771:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773938.088816:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773938.088835:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773938.216783:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773938.280754:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773938.344812:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773938.408757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773938.408757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773938.408772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773938.408802:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773938.472763:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773938.472777:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773938.472876:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773938.600780:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773938.600818:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773938.664869:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773938.728764:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773938.792756:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773938.792756:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773938.792756:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773938.856764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773938.920758:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773938.920791:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773938.920804:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773939.048765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773939.048765:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773939.048765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773939.048932:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773939.112765:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773939.112765:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773939.112765:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773939.112769:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773939.112810:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773939.112811:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773939.240749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773939.304827:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773939.368792:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773939.432814:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773939.432824:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773939.432825:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773939.432829:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773939.496870:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773939.496871:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773939.496896:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773939.624803:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773939.624809:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773939.688784:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773939.752819:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773939.816814:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773939.816817:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773939.816818:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773939.880732:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773939.944841:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773939.944842:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773939.944855:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773940.072844:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773940.072846:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773940.072847:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773940.072887:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773940.136827:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773940.136831:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773940.136837:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773940.136837:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773940.136878:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773940.136890:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773940.264850:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773940.328827:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773940.392818:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773940.392819:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773940.392831:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773940.392838:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773940.392850:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2936 00000100:00080000:1.0:1790773940.392864:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773940.392880:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773940.392929:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534789120/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773940.392959:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773940.392964:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773940.392966:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773940.392970:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2936 00000100:00080000:1.0:1790773940.392975:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773940.392991:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773940.393001:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534789120:192.168.204.150@tcp:250:kworker.0 00000008:00100000:1.0:1790773940.393003:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773940.393024:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534789248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773940.393037:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773940.393042:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773940.393044:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773940.393049:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2936 00000100:00080000:1.0:1790773940.393053:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773940.393059:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773940.393065:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773940.393078:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534789376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790773940.393086:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773940.393094:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773940.393099:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773940.393101:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773940.393107:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2936 00000100:00080000:1.0:1790773940.393112:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773940.393120:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773940.393126:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773940.393140:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534789504/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773940.393146:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773940/real 1790773940] req@ffff9c4efac4df80 x1877759534789120/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773940.393157:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773940.393163:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773940.393165:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773940.393171:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2936 00000100:00100000:2.0:1790773940.393176:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534789120/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773940.393178:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773940.393187:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773940.393194:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773940.393195:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773940.393210:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534789632/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773940.393211:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773940.393221:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534789120:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773940.393228:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773940.393234:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773940.393236:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773940.393241:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2936 00000100:00080000:1.0:1790773940.393245:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773940.393251:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773940.393255:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534789248:192.168.204.150@tcp:38:kworker.0 00000008:00100000:1.0:1790773940.393258:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773940.393274:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534789760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773940.393285:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1790773940.393289:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:1.0:1790773940.393292:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773940.393296:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773940.393303:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2936 00000100:00080000:1.0:1790773940.393307:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773940.393311:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773940/real 1790773940] req@ffff9c4efac4ea00 x1877759534789248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773940.393316:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773940.393326:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773940.393334:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534789248/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773940.393341:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534789888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773940.393355:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773940.393360:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773940.393363:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:1.0:1790773940.393366:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773940.393370:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1790773940.393372:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534789248:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773940.393379:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2936 00000100:00100000:2.0:1790773940.393383:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534789376:192.168.204.150@tcp:38:kworker.0 00000100:00080000:1.0:1790773940.393393:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773940.393401:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773940.393407:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000800:00000100:2.0:1790773940.393410:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773940.393423:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534790016/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773940.393428:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773940/real 1790773940] req@ffff9c4efac4fb80 x1877759534789376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773940.393443:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773940.393449:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1790773940.393450:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534789376/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773940.393452:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773940.393461:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2936 00000100:00080000:1.0:1790773940.393469:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773940.393477:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773940.393480:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773940.393484:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:2.0:1790773940.393485:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773940.393491:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534789376:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773940.393497:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534789504:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773940.393501:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534790144/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790773940.393517:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773940.393527:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773940/real 1790773940] req@ffff9c4efac4dc00 x1877759534789504/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773940.393535:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534789504/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773940.393542:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773940.393544:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773940.393546:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534789504:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773940.393550:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534789632:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773940.393560:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773940.393565:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773940/real 1790773940] req@ffff9c4efac4e680 x1877759534789632/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773940.393572:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534789632/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773940.393578:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773940.393580:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773940.393582:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534789632:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773940.393598:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534789760:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773940.393616:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773940.393627:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773940/real 1790773940] req@ffff9c4efac4f800 x1877759534789760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773940.393639:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534789760/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773940.393674:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773940.393679:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773940.393683:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534789760:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773940.393689:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534789888:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773940.393708:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773940.393716:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773940/real 1790773940] req@ffff9c4efac4f480 x1877759534789888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773940.393723:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534789888/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773940.393730:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773940.393733:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773940.393736:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534789888:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773940.393742:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534790016:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773940.393774:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773940.393786:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773940/real 1790773940] req@ffff9c4efac4ed80 x1877759534790016/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773940.393797:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534790016/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773940.393806:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773940.393808:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773940.393810:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534790016:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773940.393814:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534790144:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773940.393829:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773940.393835:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773940/real 1790773940] req@ffff9c4efac4c700 x1877759534790144/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773940.393842:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534790144/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773940.393848:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773940.393850:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773940.393852:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534790144:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773940.456813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773940.456814:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773940.456840:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773940.457061:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773940.520795:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773940.520795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773940.520871:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773940.648808:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773940.648808:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773940.712837:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773940.776819:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773940.840857:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773940.840858:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773940.840858:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773940.904714:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773940.968760:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773940.968760:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773940.968770:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773941.096774:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773941.096774:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773941.096796:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773941.096917:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773941.160786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773941.160786:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773941.160786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773941.160908:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773941.160920:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773941.160934:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773941.288794:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773941.352772:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773941.416778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773941.480842:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773941.480842:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773941.480842:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773941.480892:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00010000:00010000:2.0:1790773941.544945:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773941.545854:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773941.545889:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773941.672753:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773941.672758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773941.736854:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773941.800780:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773941.864795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773941.864802:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773941.865489:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773941.928774:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773941.992787:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773941.992863:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773941.992880:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773942.120778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773942.120789:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773942.120810:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773942.120855:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773942.184784:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773942.184784:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773942.184784:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773942.184810:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773942.184823:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773942.184849:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773942.312774:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773942.376780:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773942.440849:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773942.504773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773942.504774:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773942.504799:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773942.504827:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773942.568773:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773942.568788:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773942.568868:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773942.696809:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773942.696810:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773942.760798:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773942.824747:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773942.888862:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773942.888888:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773942.888894:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773942.952765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773943.016762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773943.016762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773943.016775:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773943.144798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773943.144984:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773943.147568:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773943.147605:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773943.208783:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773943.208783:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773943.208783:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773943.208907:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773943.208923:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773943.208937:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773943.336781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773943.400816:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773943.464776:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773943.528773:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773943.528773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773943.528773:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773943.528786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773943.592777:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773943.592778:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773943.592892:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773943.720795:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773943.720802:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773943.784800:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773943.848790:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773943.912778:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773943.912778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773943.912781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773943.976790:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773944.040782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773944.040782:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773944.040782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773944.168788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773944.168788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773944.168788:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773944.168817:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773944.232805:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773944.232805:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773944.232813:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773944.232846:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773944.232847:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773944.232862:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773944.360777:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773944.424853:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773944.488853:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773944.552782:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773944.552782:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773944.552782:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773944.552961:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773944.616841:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773944.616941:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773944.616967:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773944.744758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773944.745099:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773944.808841:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773944.872875:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773944.936792:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773944.936792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773944.936792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773945.000906:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773945.064795:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773945.064795:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773945.064799:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773945.192756:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773945.192756:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773945.192756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773945.192779:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773945.256759:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773945.256764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773945.256766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773945.256797:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773945.256797:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773945.256798:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773945.384764:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773945.448765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773945.512853:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773945.513805:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773945.513812:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773945.513816:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773945.513825:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2941 00000100:00080000:3.0:1790773945.513835:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773945.513845:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773945.513879:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534790272/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773945.513905:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773945.513908:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773945.513910:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773945.513923:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2941 00000100:00080000:3.0:1790773945.513927:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773945.513931:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773945.513942:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773945.513943:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534790272:192.168.204.150@tcp:250:kworker.0 00000100:00100000:3.0:1790773945.513958:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534790400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773945.513968:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773945.513972:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773945.513974:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773945.513978:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2941 00000100:00080000:3.0:1790773945.513981:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773945.513998:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773945.514003:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790773945.514014:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773945.514015:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534790528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773945.514025:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773945.514028:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773945.514030:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773945.514034:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2941 00000100:00080000:3.0:1790773945.514038:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773945.514043:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773945.514047:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773945.514058:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534790656/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773945.514064:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773945/real 1790773945] req@ffff9c4ef73c0000 x1877759534790272/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773945.514069:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773945.514077:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773945.514079:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773945.514084:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2941 00000100:00080000:3.0:1790773945.514087:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773945.514093:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773945.514104:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534790272/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773945.514107:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00080000:2.0:1790773945.514120:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1790773945.514122:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534790784/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773945.514128:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773945.514136:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534790272:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773945.514137:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773945.514145:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773945.514147:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773945.514152:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2941 00000100:00080000:3.0:1790773945.514156:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773945.514160:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773945.514166:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773945.514167:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534790400:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773945.514178:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534790912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773945.514188:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773945.514192:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773945.514193:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1790773945.514199:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773945.514202:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2941 00000100:00080000:3.0:1790773945.514207:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773945.514213:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773945.514217:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773945.514219:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773945/real 1790773945] req@ffff9c4ef73c1500 x1877759534790400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773945.514229:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534791040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773945.514238:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534790400/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773945.514240:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773945.514247:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773945.514252:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773945.514260:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2941 00000100:00080000:3.0:1790773945.514264:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773945.514269:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773945.514270:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773945.514275:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:2.0:1790773945.514277:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:3.0:1790773945.514286:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534791168/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773945.514293:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773945.514297:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1790773945.514301:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534790400:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773945.514302:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773945.514310:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2941 00000100:00100000:2.0:1790773945.514314:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534790528:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773945.514316:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773945.514325:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773945.514330:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773945.514341:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534791296/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1790773945.514349:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773945.514367:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773945/real 1790773945] req@ffff9c4ef73c0700 x1877759534790528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773945.514382:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534790528/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773945.514392:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773945.514395:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773945.514399:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534790528:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773945.514405:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534790656:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773945.514426:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773945.514438:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773945/real 1790773945] req@ffff9c4ef73c2a00 x1877759534790656/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773945.514451:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534790656/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773945.514461:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773945.514464:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773945.514467:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534790656:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773945.514472:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534790784:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773945.514490:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773945.514500:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773945/real 1790773945] req@ffff9c4ef73c2d80 x1877759534790784/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773945.514529:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534790784/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773945.514539:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773945.514543:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773945.514547:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534790784:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773945.514566:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534790912:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773945.514584:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773945.514595:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773945/real 1790773945] req@ffff9c4ef73c3800 x1877759534790912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773945.514609:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534790912/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773945.514626:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773945.514630:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773945.514633:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534790912:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773945.514639:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534791040:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773945.514696:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773945.514707:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773945/real 1790773945] req@ffff9c4ef73c0a80 x1877759534791040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773945.514719:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534791040/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773945.514732:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773945.514745:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773945.514748:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534791040:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773945.514753:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534791168:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773945.514769:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773945.514780:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773945/real 1790773945] req@ffff9c4ef73c0e00 x1877759534791168/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773945.514806:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534791168/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773945.514816:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773945.514819:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773945.514823:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534791168:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773945.514828:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534791296:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773945.514856:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773945.514868:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773945/real 1790773945] req@ffff9c4ef73c2300 x1877759534791296/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773945.514878:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534791296/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773945.514888:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773945.514891:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773945.514896:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534791296:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773945.576748:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773945.576748:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773945.576755:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773945.576784:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773945.640773:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773945.640846:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773945.640897:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773945.768778:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773945.768778:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773945.832822:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773945.896910:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773945.960803:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773945.960808:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773945.960809:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773946.024797:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773946.088827:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773946.088829:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773946.088834:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773946.216760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773946.216760:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773946.216777:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773946.216818:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773946.280757:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773946.280776:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773946.280777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773946.280777:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773946.280796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773946.280807:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773946.408781:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773946.472805:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773946.536803:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773946.600772:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773946.600772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773946.600799:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773946.600830:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773946.664805:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773946.664834:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773946.664920:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773946.792842:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773946.793436:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773946.856748:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773946.920802:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773946.984792:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773946.984794:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773946.984805:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773947.048750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773947.112813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773947.112814:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773947.112814:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773947.240782:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773947.240787:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773947.240833:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773947.240857:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773947.304796:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773947.304796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773947.304840:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773947.304903:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773947.304945:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773947.305001:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773947.432848:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773947.496831:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773947.560812:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773947.624790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773947.624918:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773947.624924:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773947.625279:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00010000:00010000:2.0:1790773947.688909:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773947.689184:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773947.689263:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773947.816803:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773947.817465:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773947.880833:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773947.944794:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773948.008762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773948.008779:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773948.008913:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773948.072765:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773948.136765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773948.136766:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773948.136774:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773948.264755:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773948.264756:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773948.264762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773948.264778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773948.328795:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773948.328796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773948.328796:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773948.328796:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773948.328862:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773948.328864:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773948.456797:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773948.520846:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773948.584816:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773948.648775:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773948.648775:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773948.648776:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773948.648918:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773948.712735:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773948.712755:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773948.712834:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773948.840768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773948.840768:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773948.904803:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773948.968761:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773949.032765:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773949.032765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773949.032810:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773949.096804:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773949.160813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773949.160813:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773949.160877:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773949.288759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773949.288759:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773949.288765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773949.288798:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773949.352773:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773949.352773:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773949.352773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773949.352791:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773949.352807:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773949.352809:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773949.480791:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773949.544816:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773949.608781:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773949.672796:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773949.672812:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773949.672856:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773949.672857:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773949.736829:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773949.736837:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773949.736868:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773949.865142:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773949.865145:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773949.928756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773949.992977:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773950.056775:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773950.056790:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773950.056790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773950.120766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773950.184813:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773950.184939:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773950.185069:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773950.312778:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773950.312778:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773950.312778:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773950.312857:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773950.376802:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773950.376803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773950.376807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773950.376838:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773950.376843:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773950.376853:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773950.504818:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773950.568784:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773950.632793:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773950.632803:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773950.632808:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773950.632818:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2946 00000100:00080000:3.0:1790773950.632829:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773950.632842:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773950.632881:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534791424/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773950.632892:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773950.632906:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773950.632913:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773950.632917:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773950.632924:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2946 00000100:00080000:3.0:1790773950.632930:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773950.632934:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534791424:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773950.632940:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773950.632952:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773950.632968:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534791552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773950.632979:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773950.632982:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773950.632984:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773950.632989:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2946 00000100:00080000:3.0:1790773950.632993:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773950.632998:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:1.0:1790773950.633000:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790773950.633006:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773950.633017:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534791680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773950.633026:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773950.633029:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773950.633032:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773950.633036:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2946 00000100:00080000:3.0:1790773950.633040:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773950.633043:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773950/real 1790773950] req@ffff9c4ef73c1c00 x1877759534791424/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773950.633047:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773950.633055:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773950.633065:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534791424/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773950.633068:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534791808/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773950.633079:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773950.633085:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773950.633086:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773950.633093:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534791424:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773950.633093:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773950.633097:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773950.633103:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2946 00000100:00080000:3.0:1790773950.633107:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773950.633111:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773950.633116:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773950.633117:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534791552:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773950.633131:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534791936/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773950.633140:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773950.633145:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773950.633150:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1790773950.633153:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773950/real 1790773950] req@ffff9c4ef73c2300 x1877759534791552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773950.633154:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773950.633164:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2946 00000100:00100000:1.0:1790773950.633168:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534791552/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773950.633170:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773950.633181:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773950.633187:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773950.633188:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773950.633194:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773950.633197:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534791552:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773950.633205:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534791680:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773950.633206:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534792064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773950.633219:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790773950.633221:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773950.633225:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773950.633228:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790773950.633232:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773950/real 1790773950] req@ffff9c4ef73c0e00 x1877759534791680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773950.633237:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2946 00000100:00080000:3.0:1790773950.633244:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773950.633249:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534791680/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773950.633253:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773950.633259:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773950.633262:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773950.633264:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773950.633269:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534791680:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773950.633274:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534791808:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773950.633279:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534792192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773950.633290:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773950.633290:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773950.633295:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773950.633298:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773950.633304:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2946 00000100:00100000:1.0:1790773950.633307:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773950/real 1790773950] req@ffff9c4ef73c0a80 x1877759534791808/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773950.633310:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773950.633319:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773950.633325:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773950.633327:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534791808/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773950.633339:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534792320/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773950.633340:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773950.633350:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773950.633359:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534791808:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790773950.633360:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773950.633367:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773950.633370:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773950.633376:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2946 00000100:00080000:3.0:1790773950.633379:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773950.633384:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534791936:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790773950.633386:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773950.633395:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773950.633409:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534792448/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773950.633418:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773950.633434:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773950/real 1790773950] req@ffff9c4ef73c3800 x1877759534791936/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773950.633447:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534791936/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773950.633456:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773950.633459:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773950.633462:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534791936:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773950.633469:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534792064:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773950.633499:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773950.633509:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773950/real 1790773950] req@ffff9c4ef73c2d80 x1877759534792064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773950.633520:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534792064/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773950.633535:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773950.633538:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773950.633542:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534792064:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773950.633564:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534792192:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773950.633585:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773950.633597:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773950/real 1790773950] req@ffff9c4ef73c2a00 x1877759534792192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773950.633615:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534792192/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773950.633623:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773950.633627:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773950.633630:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534792192:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773950.633637:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534792320:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773950.633678:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773950.633700:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773950/real 1790773950] req@ffff9c4ef73c0700 x1877759534792320/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773950.633712:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534792320/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773950.633723:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773950.633727:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773950.633730:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534792320:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773950.633747:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534792448:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773950.633765:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773950.633773:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773950/real 1790773950] req@ffff9c4ef73c1500 x1877759534792448/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773950.633781:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534792448/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773950.633791:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773950.633793:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773950.633795:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534792448:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773950.696825:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773950.696825:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773950.697002:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773950.697024:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773950.760824:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773950.760875:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773950.760921:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773950.888769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773950.888769:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773950.952769:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773951.016772:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773951.080773:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773951.080774:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773951.080884:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773951.144786:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773951.208836:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773951.208964:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773951.208976:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773951.336786:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773951.336788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773951.336793:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773951.336832:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773951.400789:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773951.400789:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773951.400792:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773951.400819:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773951.400834:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773951.400850:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773951.528740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773951.592755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773951.656760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773951.720759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773951.720767:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773951.720792:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773951.720923:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773951.784753:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773951.784753:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773951.784811:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773951.912801:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773951.912807:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773951.976782:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773952.040867:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773952.104751:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773952.104757:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773952.104772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773952.168803:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773952.232757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773952.232757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773952.232779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773952.360788:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773952.360788:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773952.360818:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773952.360857:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773952.424809:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773952.424809:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773952.424809:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773952.424815:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773952.424848:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773952.424849:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773952.552810:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773952.616800:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773952.680757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773952.744757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773952.744773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773952.744823:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773952.744971:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773952.808792:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773952.808795:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773952.808833:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773952.936812:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773952.936814:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773953.000831:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773953.064789:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773953.128800:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773953.128821:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773953.128935:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773953.192798:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773953.256747:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773953.256747:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773953.256847:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773953.384745:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773953.384745:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773953.384748:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773953.384792:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773953.448801:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773953.448804:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773953.448840:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773953.448848:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773953.448859:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773953.448889:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773953.576836:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773953.640824:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773953.704839:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773953.768788:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773953.768788:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773953.768792:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773953.768809:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773953.832782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773953.832783:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773953.832884:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773953.960730:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773953.960745:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773954.024790:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773954.088792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773954.152799:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773954.152799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773954.152803:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773954.216823:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773954.280785:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773954.280786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773954.280789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773954.408801:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773954.408801:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773954.408802:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773954.408802:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773954.472722:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773954.472738:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773954.472785:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773954.472822:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773954.472823:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773954.472844:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773954.600740:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773954.664726:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:2.0:1790773954.728774:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773954.792772:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773954.792772:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773954.792778:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773954.792778:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773954.856795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773954.856848:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773954.856857:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773954.984734:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773954.984734:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773955.048756:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773955.112754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773955.176785:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773955.176786:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773955.176786:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773955.240751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773955.304737:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773955.304744:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773955.304745:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773955.432897:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773955.432918:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773955.433287:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773955.433363:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773955.496730:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773955.496736:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773955.496800:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773955.496816:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773955.496835:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773955.496854:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773955.624797:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773955.688796:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773955.752744:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773955.752755:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773955.752761:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773955.752775:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2951 00000100:00080000:1.0:1790773955.752787:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1790773955.752788:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773955.752800:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773955.752832:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e300 x1877759534792576/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773955.752853:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773955.752857:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773955.752859:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773955.752863:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2951 00000100:00080000:1.0:1790773955.752867:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773955.752872:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773955.752881:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773955.752891:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534792704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773955.752903:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773955.752907:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773955.752908:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773955.752911:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2951 00000100:00080000:1.0:1790773955.752916:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773955.752919:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534792576:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773955.752920:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773955.752925:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773955.752939:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534792832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773955.752945:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773955.752947:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773955.752948:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773955.752951:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2951 00000100:00080000:1.0:1790773955.752952:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773955.752955:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773955.752958:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773955.752965:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534792960/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773955.752971:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773955.752973:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773955.752975:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773955.752978:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2951 00000100:00080000:1.0:1790773955.752980:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773955.752985:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773955.752989:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773955.752995:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534793088/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773955.753001:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773955.753004:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773955.753005:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773955.753008:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2951 00000100:00080000:1.0:1790773955.753010:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773955.753012:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773955.753016:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1790773955.753021:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773955.753028:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534793216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773955.753039:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773955.753043:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773955.753046:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773955.753051:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2951 00000100:00080000:1.0:1790773955.753055:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773955.753061:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773955.753068:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773955.753076:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773955/real 1790773955] req@ffff9c4efac4e300 x1877759534792576/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773955.753082:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534793344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773955.753107:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773955.753110:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e300 x1877759534792576/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773955.753111:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773955.753116:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773955.753124:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2951 00000100:00080000:1.0:1790773955.753133:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773955.753144:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773955.753149:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1790773955.753150:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00080000:2.0:1790773955.753166:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773955.753171:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534793472/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773955.753182:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534792576:192.168.204.150@tcp:250:kworker.0 00000100:00080000:1.0:1790773955.753183:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773955.753196:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773955.753199:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773955.753203:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2951 00000100:00080000:1.0:1790773955.753206:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773955.753210:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773955.753213:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773955.753222:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534793600/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773955.753228:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534792704:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773955.753257:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773955.753273:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773955/real 1790773955] req@ffff9c4efac4c700 x1877759534792704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773955.753283:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534792704/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773955.753301:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773955.753304:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773955.753306:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534792704:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773955.753312:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534792832:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773955.753331:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773955.753342:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773955/real 1790773955] req@ffff9c4efac4ed80 x1877759534792832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773955.753355:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534792832/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773955.753364:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773955.753367:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773955.753371:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534792832:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773955.753376:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534792960:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773955.753393:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773955.753400:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773955/real 1790773955] req@ffff9c4efac4f480 x1877759534792960/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773955.753408:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534792960/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773955.753415:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773955.753417:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773955.753421:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534792960:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773955.753428:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534793088:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773955.753445:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773955.753473:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773955/real 1790773955] req@ffff9c4efac4f800 x1877759534793088/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773955.753485:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534793088/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773955.753494:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773955.753497:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773955.753500:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534793088:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773955.753507:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534793216:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773955.753525:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773955.753536:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773955/real 1790773955] req@ffff9c4efac4e680 x1877759534793216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773955.753548:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534793216/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773955.753565:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773955.753569:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773955.753572:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534793216:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773955.753580:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534793344:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773955.753600:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773955.753613:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773955/real 1790773955] req@ffff9c4efac4dc00 x1877759534793344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773955.753625:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534793344/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773955.753634:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773955.753638:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773955.753641:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534793344:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773955.753648:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534793472:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773955.753694:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773955.753706:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773955/real 1790773955] req@ffff9c4efac4fb80 x1877759534793472/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773955.753718:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534793472/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773955.753730:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773955.753734:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773955.753737:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534793472:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773955.753763:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534793600:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773955.753783:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773955.753795:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773955/real 1790773955] req@ffff9c4efac4ea00 x1877759534793600/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773955.753807:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534793600/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773955.753817:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773955.753821:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773955.753825:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534793600:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773955.816786:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773955.816789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773955.816789:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773955.816838:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773955.881066:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773955.881756:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773955.883673:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773956.008758:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773956.008820:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773956.072926:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773956.136759:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773956.200769:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773956.201358:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773956.201371:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773956.264772:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773956.328771:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773956.328771:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773956.328842:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773956.456806:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773956.456870:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773956.456882:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773956.457604:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773956.520826:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773956.520893:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773956.520895:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773956.520926:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773956.521054:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773956.526703:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773956.648817:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773956.712792:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773956.777112:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773956.840772:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773956.840777:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773956.840862:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773956.840888:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773956.904773:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773956.905057:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773956.905145:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773957.032865:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773957.032868:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773957.096811:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773957.160734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773957.224789:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773957.224789:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773957.224822:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773957.288771:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773957.352757:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773957.352757:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773957.352769:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773957.480815:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773957.480816:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773957.481117:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773957.481459:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773957.544778:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773957.544778:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773957.544778:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773957.544810:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773957.545237:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773957.545259:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773957.672813:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773957.736804:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773957.800833:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773957.864769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773957.864770:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773957.864903:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773957.864910:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773957.928768:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773957.928799:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773957.928818:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773958.056773:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773958.056781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773958.120768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773958.184725:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773958.248761:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773958.248762:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773958.248923:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773958.312736:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773958.376762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773958.376762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773958.376772:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773958.504771:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773958.504797:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773958.504807:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773958.504835:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773958.568746:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773958.568754:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773958.568759:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773958.568780:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773958.568794:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773958.568802:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773958.696769:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773958.760777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773958.824756:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773958.888725:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773958.888728:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773958.888729:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773958.888729:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773958.952748:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773958.952759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773958.952829:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773959.080722:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773959.080736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773959.144734:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773959.208848:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773959.272816:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773959.272817:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773959.272825:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773959.336754:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773959.400779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773959.400780:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773959.400780:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773959.528765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773959.528765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773959.528798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773959.528805:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773959.592811:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773959.592830:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773959.592831:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773959.592832:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773959.592887:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773959.592897:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773959.720720:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773959.784845:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:3.0:1790773959.848785:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773959.912763:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773959.912791:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773959.912792:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773959.912970:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773959.976825:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773959.976883:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773959.977050:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773960.104788:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773960.104809:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773960.168791:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773960.232810:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773960.296780:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773960.296780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773960.296792:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773960.361135:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773960.424785:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773960.424785:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773960.424785:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773960.552811:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773960.552811:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773960.552812:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773960.552820:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773960.616807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773960.616807:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773960.616813:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773960.616835:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773960.616849:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773960.616855:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773960.744794:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773960.808745:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790773960.872770:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773960.872780:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773960.872785:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773960.872795:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2956 00000100:00080000:3.0:1790773960.872806:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773960.872817:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773960.872831:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773960.872849:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534793728/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773960.872871:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773960.872873:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773960.872875:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773960.872878:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2956 00000100:00080000:3.0:1790773960.872880:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773960.872883:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773960.872890:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773960.872900:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534793856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773960.872910:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773960.872914:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773960.872915:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773960.872927:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2956 00000100:00100000:2.0:1790773960.872929:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534793728:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773960.872930:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773960.872939:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773960.872947:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773960.872958:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534793984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773960.872964:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773960.872967:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773960.872968:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773960.872971:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2956 00000100:00080000:3.0:1790773960.872973:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773960.872979:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773960.872983:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773960.872990:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534794112/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773960.872996:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773960.872998:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773960.873000:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773960.873004:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2956 00000100:00080000:3.0:1790773960.873007:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773960.873010:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773960.873014:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000800:00000100:2.0:1790773960.873019:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773960.873024:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534794240/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773960.873032:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773960.873036:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773960.873039:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773960.873045:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2956 00000100:00080000:3.0:1790773960.873049:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773960.873056:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773960.873062:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773960.873070:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773960/real 1790773960] req@ffff9c4ef73c0000 x1877759534793728/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773960.873081:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534794368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773960.873093:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534793728/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773960.873095:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773960.873114:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773960.873122:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773960.873125:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773960.873132:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2956 00000100:00080000:2.0:1790773960.873136:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773960.873138:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1790773960.873144:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534793728:192.168.204.150@tcp:250:kworker.0 00000100:00080000:3.0:1790773960.873149:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773960.873158:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773960.873170:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534794496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773960.873182:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1790773960.873183:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534793856:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773960.873188:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773960.873194:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773960.873201:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2956 00000100:00080000:3.0:1790773960.873207:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773960.873220:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000800:00000100:2.0:1790773960.873221:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000008:00100000:3.0:1790773960.873231:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773960.873238:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773960/real 1790773960] req@ffff9c4ef73c1500 x1877759534793856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1790773960.873242:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534794624/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773960.873277:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534793856/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773960.873279:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773960.873293:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773960.873301:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773960.873314:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2956 00000100:00080000:3.0:1790773960.873320:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773960.873328:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:2.0:1790773960.873330:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1790773960.873332:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:2.0:1790773960.873335:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773960.873342:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534793856:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773960.873361:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534794752/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790773960.873372:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534793984:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773960.873395:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773960.873405:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773960/real 1790773960] req@ffff9c4ef73c0700 x1877759534793984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773960.873413:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534793984/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773960.873420:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773960.873423:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773960.873427:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534793984:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773960.873434:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534794112:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773960.873454:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773960.873465:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773960/real 1790773960] req@ffff9c4ef73c2a00 x1877759534794112/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773960.873484:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534794112/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773960.873495:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773960.873499:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773960.873502:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534794112:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773960.873508:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534794240:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773960.873529:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773960.873540:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773960/real 1790773960] req@ffff9c4ef73c2d80 x1877759534794240/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773960.873552:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534794240/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773960.873561:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773960.873563:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773960.873566:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534794240:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773960.873570:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534794368:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773960.873598:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773960.873611:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773960/real 1790773960] req@ffff9c4ef73c3800 x1877759534794368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773960.873623:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534794368/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773960.873644:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773960.873649:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773960.873682:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534794368:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773960.873702:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534794496:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773960.873726:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773960.873737:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773960/real 1790773960] req@ffff9c4ef73c0a80 x1877759534794496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773960.873745:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534794496/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773960.873750:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773960.873753:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773960.873755:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534794496:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773960.873758:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534794624:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773960.873769:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773960.873775:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773960/real 1790773960] req@ffff9c4ef73c0e00 x1877759534794624/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773960.873782:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534794624/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773960.873788:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773960.873790:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773960.873793:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534794624:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773960.873797:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534794752:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773960.873807:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773960.873812:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773960/real 1790773960] req@ffff9c4ef73c2300 x1877759534794752/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773960.873819:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534794752/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773960.873826:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773960.873827:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773960.873829:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534794752:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790773960.936759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773960.936771:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773960.936783:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773960.936817:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773961.000726:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773961.000782:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790773961.000902:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773961.128742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773961.128782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773961.192749:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773961.256804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773961.320728:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773961.320741:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773961.320782:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773961.384775:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773961.448774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773961.448784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773961.448798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773961.576736:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773961.576747:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773961.576763:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773961.576784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773961.640766:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773961.640766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773961.640766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773961.640791:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773961.640809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773961.640848:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773961.768757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773961.832735:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773961.896767:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773961.960724:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773961.960754:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773961.960757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773961.960769:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773962.024759:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773962.024806:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773962.024809:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773962.152755:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773962.152779:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773962.216800:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773962.280754:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773962.344787:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773962.344787:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773962.344849:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773962.408751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773962.472750:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773962.472756:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773962.472777:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773962.600768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773962.600768:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773962.600768:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773962.600788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773962.664751:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773962.664751:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773962.664751:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773962.664756:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773962.664783:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773962.664786:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773962.792741:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773962.856743:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773962.920740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773962.984768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773962.984777:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773962.984785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773962.984815:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773963.048794:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773963.048795:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773963.048858:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773963.176743:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773963.176743:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773963.240753:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773963.304738:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773963.368743:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773963.368749:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773963.368787:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773963.432797:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773963.496789:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773963.496789:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773963.496832:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773963.624798:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773963.624798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773963.624830:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773963.625808:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773963.688768:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773963.688769:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773963.688810:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773963.688897:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773963.688916:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773963.688939:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773963.816809:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773963.880862:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773963.944817:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773964.008812:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773964.008812:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773964.008812:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773964.008826:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773964.072800:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773964.072813:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773964.072847:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773964.200795:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773964.200844:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773964.264772:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773964.329139:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773964.392766:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773964.392766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773964.392865:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773964.456817:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773964.520773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773964.520774:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773964.520779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773964.648780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773964.648780:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773964.649172:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773964.649181:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773964.712796:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773964.712835:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773964.713161:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773964.713164:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773964.714327:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773964.714504:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773964.840772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773964.904777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773964.968778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773965.032796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773965.034249:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773965.034382:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773965.034428:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773965.096953:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773965.097067:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773965.097098:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773965.224766:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773965.224767:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773965.289955:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773965.352828:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773965.416772:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773965.416782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773965.416784:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773965.480767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773965.544874:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773965.545245:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773965.545436:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773965.672769:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773965.672769:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773965.672769:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773965.672772:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773965.736799:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773965.736799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773965.736799:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773965.736834:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773965.736884:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773965.736919:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773965.864863:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773965.928872:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773965.992807:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773965.992823:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773965.992834:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773965.992840:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773965.992854:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2961 00000100:00080000:3.0:1790773965.992868:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773965.992881:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773965.992931:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534794880/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773965.993029:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534794880:192.168.204.150@tcp:250:kworker.0 00000800:00000100:1.0:1790773965.993108:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773965.993220:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773965.993225:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773965.993227:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773965.993232:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2961 00000100:00080000:3.0:1790773965.993236:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773965.993249:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773965.993276:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773965.993292:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534795008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773965.993435:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773965/real 1790773965] req@ffff9c4ef73c1c00 x1877759534794880/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773965.993454:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534794880/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773965.993463:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773965.993466:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773965.993469:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534794880:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773965.993494:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534795008:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773965.993535:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773965.993545:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773965/real 1790773965] req@ffff9c4ef73c2300 x1877759534795008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773965.993553:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534795008/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773965.993565:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773965.993567:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773965.993572:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534795008:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773965.993572:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773965.993587:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773965.993591:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773965.993598:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2961 00000100:00080000:3.0:1790773965.993603:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773965.993611:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773965.993617:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773965.993633:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534795136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773965.993682:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773965.993687:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534795136:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773965.993691:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773965.993695:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773965.993702:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2961 00000800:00000100:1.0:1790773965.993717:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773965.993726:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773965/real 1790773965] req@ffff9c4ef73c0e00 x1877759534795136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773965.993734:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534795136/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773965.993740:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773965.993742:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773965.993744:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534795136:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773965.993987:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773965.993998:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773965.994007:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773965.994030:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534795264/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773965.994054:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773965.994058:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773965.994061:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773965.994066:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2961 00000100:00080000:3.0:1790773965.994071:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773965.994079:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773965.994080:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534795264:192.168.204.150@tcp:8:kworker.0 00000008:00100000:3.0:1790773965.994088:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773965.994104:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534795392/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773965.994105:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773965.994116:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773965/real 1790773965] req@ffff9c4ef73c0a80 x1877759534795264/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773965.994120:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773965.994127:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1790773965.994129:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534795264/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773965.994131:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773965.994139:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2961 00000100:00080000:1.0:1790773965.994141:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773965.994144:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773965.994147:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773965.994151:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534795264:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790773965.994154:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773965.994163:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534795392:192.168.204.150@tcp:8:kworker.0 00000008:00100000:3.0:1790773965.994163:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1790773965.994182:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:3.0:1790773965.994182:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534795520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773965.994191:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773965/real 1790773965] req@ffff9c4ef73c3800 x1877759534795392/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773965.994196:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773965.994201:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1790773965.994202:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534795392/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773965.994205:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773965.994214:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2961 00000100:00080000:1.0:1790773965.994215:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773965.994220:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00080000:3.0:1790773965.994221:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773965.994229:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534795392:192.168.204.150@tcp:8:kworker.0 00000100:00080000:3.0:1790773965.994230:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773965.994239:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773965.994242:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534795520:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773965.994265:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534795648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773965.994275:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773965.994286:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773965/real 1790773965] req@ffff9c4ef73c2d80 x1877759534795520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773965.994290:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773965.994295:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534795520/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773965.994296:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773965.994300:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773965.994307:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2961 00000100:00080000:3.0:1790773965.994312:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773965.994313:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1790773965.994320:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00080000:1.0:1790773965.994321:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773965.994328:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534795520:192.168.204.150@tcp:38:kworker.0 00000008:00100000:3.0:1790773965.994329:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773965.994346:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534795776/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773965.994347:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534795648:192.168.204.150@tcp:38:kworker.0 00000100:00080000:3.0:1790773965.994364:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773965.994368:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773965.994370:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1790773965.994372:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:3.0:1790773965.994378:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2961 00000100:00080000:3.0:1790773965.994383:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773965.994384:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773965/real 1790773965] req@ffff9c4ef73c2a00 x1877759534795648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1790773965.994392:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773965.994395:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534795648/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1790773965.994399:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00080000:1.0:1790773965.994404:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773965.994407:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773965.994410:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534795648:192.168.204.150@tcp:38:kworker.0 00000100:00100000:3.0:1790773965.994415:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534795904/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773965.994420:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534795776:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773965.994438:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773965.994446:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773965/real 1790773965] req@ffff9c4ef73c0700 x1877759534795776/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773965.994455:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534795776/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773965.994462:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773965.994465:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773965.994467:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534795776:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773965.994472:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534795904:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773965.994481:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773965.994486:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773965/real 1790773965] req@ffff9c4ef73c1500 x1877759534795904/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773965.994493:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534795904/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773965.994499:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773965.994500:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773965.994503:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534795904:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773966.056828:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773966.057162:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773966.057183:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773966.057304:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773966.120832:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773966.120967:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773966.121363:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773966.248777:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773966.248934:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773966.313305:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773966.376905:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773966.440806:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773966.440807:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773966.441172:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773966.504803:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773966.568802:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773966.568845:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773966.568899:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773966.696798:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773966.696835:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773966.696844:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773966.696901:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773966.760801:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773966.760805:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773966.760832:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773966.760843:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773966.760866:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773966.760878:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773966.889219:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773966.952806:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773967.016777:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773967.080813:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773967.080813:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773967.080826:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773967.080859:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00010000:00010000:1.0:1790773967.144883:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773967.144938:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773967.144947:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773967.272764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773967.273129:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773967.336776:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773967.400877:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773967.464832:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773967.464862:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773967.464872:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773967.528830:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773967.592806:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773967.592807:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773967.592817:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773967.720797:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773967.720998:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773967.721356:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773967.724121:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773967.784726:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773967.784773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773967.784777:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773967.785255:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773967.785266:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773967.785355:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773967.912798:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773967.976789:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773968.040834:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773968.104797:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773968.104798:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773968.104810:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773968.104810:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773968.168833:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773968.168842:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773968.168985:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773968.296786:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773968.296786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773968.360763:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773968.424784:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773968.488796:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773968.488796:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773968.488944:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773968.552804:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773968.616754:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773968.616784:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773968.616784:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773968.744873:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773968.744875:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773968.744883:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773968.744919:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773968.808795:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773968.808795:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773968.808804:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773968.808804:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773968.808848:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773968.808862:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773968.936803:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773969.000777:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773969.064778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773969.128801:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773969.128801:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773969.128803:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773969.129044:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773969.192821:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773969.192856:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773969.193293:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773969.320817:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773969.320819:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773969.387556:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773969.448797:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773969.512780:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773969.512781:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773969.512879:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773969.576781:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773969.640841:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773969.640841:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773969.640843:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773969.768783:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773969.768784:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773969.769128:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773969.769221:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773969.832765:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773969.832765:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773969.832796:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773969.832808:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773969.832814:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773969.833710:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773969.960772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773970.024764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773970.088770:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773970.152781:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773970.152781:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773970.152800:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773970.153015:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773970.216810:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773970.216823:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773970.216843:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773970.344813:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773970.345011:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773970.409002:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773970.474711:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773970.536740:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773970.536740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773970.536743:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773970.601586:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773970.664820:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773970.664951:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773970.665148:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773970.792759:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773970.792759:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773970.792762:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773970.792817:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773970.856774:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773970.856784:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773970.856814:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773970.856929:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773970.856945:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773970.856986:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773970.984767:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773971.048835:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773971.112829:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773971.112842:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773971.112848:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773971.112858:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2967 00000100:00080000:2.0:1790773971.112869:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773971.112881:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773971.112918:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534796032/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773971.112943:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773971.112947:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773971.112949:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773971.112954:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2967 00000100:00080000:2.0:1790773971.112958:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773971.112963:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773971.112972:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773971.112985:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534796160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773971.112995:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773971.112999:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773971.113001:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773971.113005:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2967 00000100:00080000:2.0:1790773971.113008:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773971.113012:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773971.113017:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773971.113029:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534796288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773971.113044:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773971.113047:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773971.113049:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773971.113054:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2967 00000100:00080000:2.0:1790773971.113057:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773971.113061:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773971.113066:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773971.113075:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534796416/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773971.113084:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773971.113087:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773971.113089:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773971.113093:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2967 00000100:00080000:2.0:1790773971.113096:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773971.113100:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773971.113105:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773971.113113:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534796544/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773971.113122:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773971.113126:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773971.113128:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773971.113132:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2967 00000100:00080000:2.0:1790773971.113135:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773971.113140:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773971.113145:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773971.113169:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534796672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773971.113178:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773971.113182:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773971.113184:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773971.113188:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2967 00000100:00080000:2.0:1790773971.113193:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773971.113198:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773971.113203:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773971.113214:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534796800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773971.113223:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773971.113226:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773971.113228:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773971.113232:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2967 00000100:00080000:2.0:1790773971.113236:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773971.113241:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773971.113246:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773971.113255:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534796928/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773971.113261:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534796032:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773971.113278:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773971.113280:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773971.113282:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773971.113287:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2967 00000100:00080000:2.0:1790773971.113290:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773971.113295:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773971.113301:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773971.113312:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534797056/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773971.113334:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773971.113382:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773971/real 1790773971] req@ffff9c4efbcc2300 x1877759534796032/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773971.113407:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534796032/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773971.113410:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773971.113426:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773971.113433:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773971.113439:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534796032:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773971.113450:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534796160:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773971.113475:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773971.113489:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773971/real 1790773971] req@ffff9c4efbcc0380 x1877759534796160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773971.113501:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534796160/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773971.113518:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773971.113522:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773971.113526:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534796160:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773971.113532:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534796288:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773971.113552:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773971.113564:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773971/real 1790773971] req@ffff9c4efbcc1f80 x1877759534796288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773971.113575:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534796288/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773971.113584:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773971.113588:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773971.113592:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534796288:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773971.113598:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534796416:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773971.113620:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773971.113631:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773971/real 1790773971] req@ffff9c4efbcc1c00 x1877759534796416/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773971.113644:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534796416/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773971.113677:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773971.113682:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773971.113686:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534796416:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773971.113692:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534796544:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773971.113715:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773971.113733:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773971/real 1790773971] req@ffff9c4efbcc3100 x1877759534796544/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773971.113747:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534796544/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773971.113757:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773971.113761:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773971.113764:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534796544:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773971.113770:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534796672:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773971.113790:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773971.113801:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773971/real 1790773971] req@ffff9c4efbcc2680 x1877759534796672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773971.113820:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534796672/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773971.113836:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773971.113839:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773971.113843:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534796672:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773971.113848:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534796800:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773971.113869:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773971.113880:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773971/real 1790773971] req@ffff9c4efbcc1880 x1877759534796800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773971.113890:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534796800/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773971.113898:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773971.113902:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773971.113905:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534796800:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773971.113938:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534796928:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773971.113958:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773971.113969:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773971/real 1790773971] req@ffff9c4efbcc1500 x1877759534796928/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773971.113981:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534796928/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773971.113996:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773971.113999:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773971.114003:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534796928:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773971.114009:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534797056:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773971.114027:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773971.114038:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773971/real 1790773971] req@ffff9c4efbcc0700 x1877759534797056/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773971.114049:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534797056/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790773996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773971.114059:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773971.114062:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773971.114065:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534797056:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773971.176757:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773971.176757:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773971.176757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773971.176762:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773971.240763:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773971.240820:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773971.240838:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773971.368736:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773971.368828:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773971.432791:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773971.496783:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773971.560794:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773971.560794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773971.560800:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773971.624766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773971.688738:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773971.688738:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773971.688741:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773971.816741:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773971.816741:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773971.816742:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773971.816773:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773971.880783:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773971.880787:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773971.880809:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773971.880832:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773971.880944:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773971.880960:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773972.008766:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773972.072814:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773972.136778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773972.200747:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773972.200747:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773972.200747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773972.200750:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773972.264743:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773972.264758:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773972.264813:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773972.392749:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773972.392749:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773972.456777:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773972.520726:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773972.584720:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773972.584724:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773972.584859:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773972.648750:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773972.712740:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773972.712740:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773972.712744:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773972.840767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773972.840768:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773972.840768:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773972.840910:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773972.904744:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773972.904744:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773972.904746:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773972.904875:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773972.904924:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773972.904953:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773973.032752:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773973.096790:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773973.160750:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773973.224739:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773973.224739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773973.224740:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773973.224753:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773973.288724:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773973.288750:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773973.288800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773973.416720:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773973.416728:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773973.480760:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773973.544842:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773973.608796:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773973.608796:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773973.608808:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773973.672759:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773973.736743:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773973.736743:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773973.736791:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773973.864729:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773973.864731:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773973.864791:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773973.864791:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773973.928756:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773973.928756:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773973.928756:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773973.928882:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773973.928883:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773973.928942:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773974.056750:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773974.120762:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773974.184765:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773974.248726:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773974.248726:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773974.248733:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773974.248830:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773974.312742:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773974.312775:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773974.312906:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773974.440721:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773974.440721:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773974.504716:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773974.568734:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773974.632769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773974.632769:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773974.632779:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773974.696766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773974.760749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773974.760749:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773974.760750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773974.888749:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773974.888754:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773974.890171:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773974.890247:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773974.952799:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773974.952813:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773974.952892:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773974.952967:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773974.953003:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773974.953033:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773975.080733:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773975.144785:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773975.208799:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773975.272733:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773975.272733:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773975.272733:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773975.272765:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773975.336787:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773975.336836:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773975.336858:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773975.464777:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773975.464780:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773975.528775:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773975.592888:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773975.656799:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773975.656799:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773975.656801:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773975.721024:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773975.784796:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773975.784796:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773975.784948:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773975.912793:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773975.913210:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773975.913907:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773975.913966:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773975.976757:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773975.977131:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773975.977363:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773975.978049:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773975.978116:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773975.978381:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773976.104814:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773976.168810:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:2.0:1790773976.232871:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773976.232882:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773976.232887:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773976.232897:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2972 00000100:00080000:2.0:1790773976.232908:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773976.232921:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773976.232962:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534797184/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773976.233004:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773976.233008:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773976.233010:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773976.233015:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2972 00000100:00080000:2.0:1790773976.233018:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773976.233023:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773976.233033:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773976.233047:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534797312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773976.233071:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773976.233076:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1790773976.233078:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534797184:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773976.233079:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773976.233086:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2972 00000100:00080000:2.0:1790773976.233090:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773976.233094:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773976.233098:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773976.233108:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534797440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773976.233114:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773976.233117:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773976.233118:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773976.233121:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2972 00000100:00080000:2.0:1790773976.233125:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773976.233128:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773976.233132:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773976.233140:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534797568/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773976.233147:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773976.233150:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773976.233153:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773976.233154:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773976.233158:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2972 00000100:00080000:2.0:1790773976.233160:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773976.233164:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773976.233167:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773976.233174:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534797696/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773976.233180:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773976.233183:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773976.233184:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773976.233187:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2972 00000100:00080000:2.0:1790773976.233190:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773976.233195:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773976.233196:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773976/real 1790773976] req@ffff9c4efbcc2a00 x1877759534797184/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773976.233198:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773976.233208:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534797824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773976.233220:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1790773976.233221:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534797184/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773976.233223:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773976.233225:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773976.233231:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2972 00000100:00080000:2.0:1790773976.233236:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773976.233241:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773976.233245:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1790773976.233247:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1790773976.233253:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534797952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773976.233255:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773976.233263:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534797184:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790773976.233269:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773976.233274:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773976.233276:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773976.233282:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2972 00000100:00080000:2.0:1790773976.233285:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1790773976.233292:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534797312:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790773976.233293:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773976.233302:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773976.233318:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534798080/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1790773976.233327:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00080000:2.0:1790773976.233329:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773976.233334:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773976.233337:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773976.233342:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2972 00000100:00100000:1.0:1790773976.233344:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773976/real 1790773976] req@ffff9c4efbcc0a80 x1877759534797312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773976.233348:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773976.233358:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773976.233363:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534797312/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1790773976.233364:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773976.233382:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534798208/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773976.233387:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773976.233396:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773976.233401:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534797312:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773976.233411:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534797440:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773976.233431:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773976.233442:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773976/real 1790773976] req@ffff9c4efbcc0700 x1877759534797440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773976.233455:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534797440/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773976.233463:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773976.233467:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773976.233471:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534797440:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773976.233477:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534797568:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773976.233494:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773976.233503:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773976/real 1790773976] req@ffff9c4efbcc1500 x1877759534797568/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773976.233514:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534797568/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773976.233524:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773976.233528:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773976.233531:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534797568:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773976.233537:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534797696:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773976.233554:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773976.233563:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773976/real 1790773976] req@ffff9c4efbcc1880 x1877759534797696/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773976.233574:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534797696/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773976.233583:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773976.233587:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773976.233590:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534797696:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773976.233596:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534797824:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773976.233615:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773976.233628:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773976/real 1790773976] req@ffff9c4efbcc2680 x1877759534797824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773976.233640:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534797824/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1790773976.233675:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:1.0:1790773976.234475:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773976.234481:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773976.234486:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534797824:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773976.234495:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534797952:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773976.234521:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773976.234534:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773976/real 1790773976] req@ffff9c4efbcc3100 x1877759534797952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773976.234547:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534797952/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773976.234561:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773976.234564:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773976.234568:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534797952:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773976.234593:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534798080:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773976.234611:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773976.234621:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773976/real 1790773976] req@ffff9c4efbcc1c00 x1877759534798080/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773976.234632:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534798080/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773976.234642:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773976.234646:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773976.234649:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534798080:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773976.234675:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534798208:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773976.234693:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773976.234702:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773976/real 1790773976] req@ffff9c4efbcc1f80 x1877759534798208/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773976.234713:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534798208/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773976.234722:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773976.234726:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773976.234729:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534798208:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773976.296758:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773976.296758:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773976.296878:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773976.296898:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773976.360796:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773976.360836:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773976.361079:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773976.488946:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773976.489032:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773976.552794:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773976.617755:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773976.680790:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773976.680801:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773976.680829:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773976.744796:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773976.808808:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773976.808813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773976.808817:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773976.937004:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773976.937285:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773976.937285:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773976.937471:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773977.000808:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773977.000808:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773977.000808:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773977.000859:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773977.000972:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773977.001027:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773977.128810:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773977.192799:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773977.256895:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773977.320767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773977.320774:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773977.320800:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773977.323167:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773977.384809:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773977.384815:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773977.384892:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773977.512823:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773977.513222:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773977.576993:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773977.640901:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773977.704824:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773977.704824:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773977.705006:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773977.768944:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773977.832779:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773977.832789:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773977.833691:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773977.960826:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773977.960826:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773977.961127:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773977.961295:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773978.024803:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773978.024811:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773978.025199:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773978.025226:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773978.025271:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773978.025301:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773978.152895:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773978.216824:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773978.280786:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773978.344777:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773978.344804:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773978.344839:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773978.345006:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773978.408786:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773978.408913:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773978.409232:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773978.536797:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773978.536809:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773978.600837:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773978.664855:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773978.728777:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773978.728777:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773978.728780:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773978.792780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773978.856775:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773978.856775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773978.857178:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773978.984847:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773978.984928:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773978.984939:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773978.984939:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773979.048784:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773979.048945:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773979.049110:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773979.049778:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773979.049918:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773979.049995:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773979.176862:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773979.240811:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773979.304934:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773979.368764:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773979.368764:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773979.368764:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773979.368794:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773979.432794:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773979.432800:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773979.432920:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773979.560781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773979.560781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773979.624827:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773979.688838:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773979.752763:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773979.752763:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773979.753018:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773979.817293:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773979.880815:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773979.880815:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773979.880827:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773980.008820:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773980.009237:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773980.009372:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773980.009401:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773980.072788:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773980.072856:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773980.072862:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773980.073049:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773980.073081:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773980.073423:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773980.200794:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773980.264813:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773980.328803:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773980.392913:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773980.393018:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773980.393214:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773980.393405:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00010000:00010000:1.0:1790773980.456852:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773980.457205:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773980.457236:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790773980.584764:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773980.584892:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773980.648840:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773980.712820:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773980.776750:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773980.776795:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773980.776849:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773980.840780:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773980.904818:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773980.904822:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773980.906545:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773981.032792:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773981.032844:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773981.032859:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773981.032869:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773981.096767:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773981.096767:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773981.096826:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773981.097232:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773981.097292:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773981.097450:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773981.225852:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773981.289045:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773981.353832:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:0.0:1790773981.354972:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773981.354980:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1790773981.354985:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773981.354994:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2977 00000100:00080000:0.0:1790773981.355004:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773981.355015:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:0.0:1790773981.355055:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d180 x1877759534798336/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773981.355086:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773981.355090:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773981.355092:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773981.355097:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2977 00000100:00080000:0.0:1790773981.355101:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773981.355106:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773981.355116:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773981.355129:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7f480 x1877759534798464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773981.355144:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773981.355147:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773981.355149:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773981.355153:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2977 00000100:00080000:0.0:1790773981.355156:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773981.355162:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773981.355166:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773981.355176:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ed80 x1877759534798592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773981.355185:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773981.355188:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773981.355190:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773981.355194:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2977 00000100:00080000:0.0:1790773981.355197:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773981.355202:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773981.355206:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:0.0:1790773981.355216:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7df80 x1877759534798720/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773981.355224:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773981.355227:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773981.355229:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773981.355233:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2977 00000100:00080000:0.0:1790773981.355237:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773981.355241:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773981.355245:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:0.0:1790773981.355255:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7d880 x1877759534798848/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773981.355263:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773981.355267:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1790773981.355268:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773981.355272:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2977 00000100:00080000:0.0:1790773981.355275:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773981.355279:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773981.355284:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773981.355293:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7c380 x1877759534798976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773981.355302:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773981.355305:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1790773981.355307:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773981.355311:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2977 00000100:00080000:0.0:1790773981.355315:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773981.355319:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773981.355323:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1790773981.355332:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7ca80 x1877759534799104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773981.355347:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773981.355350:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1790773981.355352:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773981.355356:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2977 00000100:00080000:0.0:1790773981.355360:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773981.355364:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773981.355369:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:0.0:1790773981.355379:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7fb80 x1877759534799232/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1790773981.355387:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1790773981.355390:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1790773981.355392:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1790773981.355395:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2977 00000100:00080000:0.0:1790773981.355398:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1790773981.355402:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:0.0:1790773981.355406:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:0.0:1790773981.355415:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ec7a7e300 x1877759534799360/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773981.355503:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534798336:192.168.204.150@tcp:250:kworker.0 00000800:00000100:1.0:1790773981.355563:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773981.355602:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773981/real 1790773981] req@ffff9c4ec7a7d180 x1877759534798336/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773981.355619:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d180 x1877759534798336/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773981.355629:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773981.355634:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773981.355638:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534798336:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773981.355645:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534798464:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773981.355689:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773981.355701:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773981/real 1790773981] req@ffff9c4ec7a7f480 x1877759534798464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773981.355713:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7f480 x1877759534798464/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773981.355728:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773981.355732:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773981.355736:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534798464:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773981.355741:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534798592:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773981.355759:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773981.355769:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773981/real 1790773981] req@ffff9c4ec7a7ed80 x1877759534798592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773981.355779:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ed80 x1877759534798592/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773981.355786:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773981.355789:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773981.355793:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534798592:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773981.355798:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534798720:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773981.355814:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773981.355823:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773981/real 1790773981] req@ffff9c4ec7a7df80 x1877759534798720/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773981.355835:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7df80 x1877759534798720/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773981.355844:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773981.355847:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773981.355850:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534798720:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773981.355856:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534798848:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773981.355872:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773981.355880:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773981/real 1790773981] req@ffff9c4ec7a7d880 x1877759534798848/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773981.355891:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7d880 x1877759534798848/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773981.355900:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773981.355904:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773981.355907:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534798848:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773981.355913:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534798976:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773981.355930:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773981.355940:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773981/real 1790773981] req@ffff9c4ec7a7c380 x1877759534798976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773981.355970:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7c380 x1877759534798976/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773981.355985:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773981.355989:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773981.355992:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534798976:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773981.355998:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534799104:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773981.356017:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773981.356028:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773981/real 1790773981] req@ffff9c4ec7a7ca80 x1877759534799104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773981.356039:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7ca80 x1877759534799104/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773981.356045:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773981.356048:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773981.356050:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534799104:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773981.356055:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534799232:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773981.356071:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773981.356082:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773981/real 1790773981] req@ffff9c4ec7a7fb80 x1877759534799232/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773981.356093:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7fb80 x1877759534799232/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773981.356108:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773981.356111:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773981.356114:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534799232:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773981.356119:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534799360:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773981.356137:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773981.356147:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773981/real 1790773981] req@ffff9c4ec7a7e300 x1877759534799360/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773981.356158:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ec7a7e300 x1877759534799360/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773981.356168:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773981.356171:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773981.356174:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ec7a7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534799360:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773981.416810:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773981.416810:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773981.419378:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773981.419499:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00010000:00010000:1.0:1790773981.480850:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773981.480972:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773981.481055:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773981.608787:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773981.608943:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773981.672827:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773981.737151:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773981.800793:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773981.804738:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773981.804778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773981.864805:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773981.928845:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773981.928864:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773981.928864:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773982.056810:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773982.056834:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773982.056874:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773982.057044:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773982.120845:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773982.120846:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773982.120847:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773982.121680:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773982.121702:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773982.121780:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773982.248788:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773982.312767:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773982.376740:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773982.440835:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773982.441815:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773982.441926:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773982.441998:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773982.504846:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773982.505091:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773982.507180:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773982.632768:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773982.632832:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773982.696785:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773982.760792:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773982.824766:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773982.824769:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773982.824876:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773982.888782:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773982.952768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773982.952768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773982.953102:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773983.080767:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773983.080767:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773983.080805:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773983.080964:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773983.144778:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773983.144785:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773983.144896:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773983.145122:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773983.145152:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773983.145626:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773983.275673:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773983.336787:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773983.400775:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773983.464777:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773983.464882:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773983.467217:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773983.467986:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773983.528851:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773983.529025:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773983.530471:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773983.657251:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773983.657362:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773983.720889:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773983.784960:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773983.850732:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773983.850888:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773983.850922:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773983.912767:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773983.976877:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773983.977216:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773983.977772:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773984.104721:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773984.104731:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773984.107515:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773984.107700:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773984.168803:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773984.169355:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773984.169383:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773984.170095:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773984.171602:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773984.171999:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773984.298856:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773984.365011:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773984.424787:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773984.489015:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773984.489188:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773984.489497:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773984.489811:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00010000:00010000:2.0:1790773984.555227:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773984.555269:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773984.555666:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773984.683612:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773984.683718:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773984.744791:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773984.810545:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773984.872771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773984.872919:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773984.873304:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773984.936815:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773985.000793:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773985.003086:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773985.004115:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773985.128803:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773985.134115:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773985.134197:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773985.134300:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773985.192781:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773985.192781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773985.192949:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773985.193000:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773985.193027:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773985.193370:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773985.320789:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773985.384760:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773985.448766:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773985.512763:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773985.513592:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773985.513612:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773985.513685:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773985.576764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790773985.576764:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773985.576999:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773985.704757:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773985.704958:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773985.769065:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773985.832804:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773985.896914:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773985.897282:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773985.897840:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773985.960852:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773986.025075:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773986.025354:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773986.027275:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773986.152837:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773986.152908:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773986.153024:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773986.153100:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773986.216767:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773986.216768:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773986.216897:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773986.217005:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773986.217153:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773986.217208:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773986.344790:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773986.408765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773986.472778:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790773986.474309:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773986.474318:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790773986.474322:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773986.474332:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2982 00000100:00080000:3.0:1790773986.474342:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773986.474353:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790773986.474391:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534799488/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773986.474425:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773986.474430:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773986.474432:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773986.474437:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2982 00000100:00080000:3.0:1790773986.474441:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773986.474445:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773986.474456:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773986.474471:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534799616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773986.474481:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773986.474485:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773986.474487:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773986.474491:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2982 00000100:00080000:3.0:1790773986.474494:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773986.474499:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773986.474503:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773986.474514:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534799744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773986.474523:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773986.474526:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773986.474528:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773986.474542:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2982 00000100:00080000:3.0:1790773986.474545:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773986.474550:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773986.474555:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790773986.474566:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534799872/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773986.474575:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773986.474579:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773986.474581:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773986.474584:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2982 00000100:00080000:3.0:1790773986.474588:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773986.474592:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773986.474597:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790773986.474606:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534800000/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773986.474614:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773986.474617:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790773986.474619:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773986.474623:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2982 00000100:00080000:3.0:1790773986.474626:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773986.474630:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773986.474634:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773986.474644:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534800128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773986.474674:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773986.474678:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790773986.474680:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773986.474685:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2982 00000100:00080000:3.0:1790773986.474689:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773986.474693:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773986.474698:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790773986.474709:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534800256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773986.474718:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773986.474721:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790773986.474723:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773986.474727:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2982 00000100:00080000:3.0:1790773986.474730:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773986.474735:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773986.474739:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790773986.474749:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534800384/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790773986.474758:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790773986.474761:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790773986.474763:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790773986.474767:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2982 00000100:00080000:3.0:1790773986.474771:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790773986.474775:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790773986.474779:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790773986.474789:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534800512/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773986.476832:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534799488:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773986.476950:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534799616:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773986.476973:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534799744:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773986.476991:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534799872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773986.477009:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534800000:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773986.477026:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534800128:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773986.477043:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534800256:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773986.477072:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534800384:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773986.477089:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534800512:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773986.536743:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773986.536743:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773986.536757:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773986.536884:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773986.600966:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773986.602235:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773986.602264:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773986.729142:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773986.730481:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773986.793080:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773986.857618:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773986.920884:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773986.920885:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773986.921021:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773986.984802:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773987.049006:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773987.049008:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773987.050015:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773987.176811:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773987.176855:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773987.176902:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773987.176948:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773987.240807:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773987.240992:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773987.241125:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773987.241317:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773987.241335:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773987.242679:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773987.370381:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773987.432796:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773987.496815:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773987.560796:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773987.560796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773987.560849:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773987.560915:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773987.624867:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773987.624998:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773987.626273:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773987.752756:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773987.755859:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773987.816784:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773987.880757:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773987.944820:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773987.944913:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773987.945004:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773988.010072:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773988.072773:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773988.072779:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773988.072890:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773988.200757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773988.200757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773988.200873:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773988.200950:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773988.264766:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773988.264766:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773988.264774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773988.264786:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773988.264814:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773988.264834:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773988.392835:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773988.456752:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773988.520841:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773988.584784:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773988.585088:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773988.585217:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773988.587160:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00010000:00010000:1.0:1790773988.648947:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773988.649022:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773988.654426:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773988.776771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773988.776877:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773988.840780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773988.904773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773988.968789:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773988.968900:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773988.968982:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773989.042686:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773989.096836:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773989.097066:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773989.097145:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773989.224782:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773989.224950:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773989.225056:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773989.225500:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773989.288795:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773989.289069:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773989.289158:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790773989.289244:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773989.289474:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773989.289572:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773989.416772:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773989.481047:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773989.544949:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000400:00000100:3.0:1790773989.545561:0:2386:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.150:988 00000400:00000100:3.0:1790773989.545601:0:2386:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.204.150@tcp at host 192.168.204.150 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1790773989.545616:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773989.545704:0:2386:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000800:00000100:3.0:1790773989.545717:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773989.545726:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773989.545737:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773989.545747:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:1.0:1790773989.545753:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773986/real 1790773989] req@ffff9c4ef73c0000 x1877759534799488/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1790773989.545759:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000800:00000100:3.0:1790773989.545771:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00100000:1.0:1790773989.545785:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534799488/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1790773989.545787:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.150@tcp 00000100:00080000:1.0:1790773989.545813:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773989.545820:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773989.545825:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534799488:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773989.545833:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773986/real 1790773989] req@ffff9c4ef73c1500 x1877759534799616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773989.545854:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534799616/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773989.545873:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773989.545877:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773989.545880:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534799616:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773989.545886:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773986/real 1790773989] req@ffff9c4ef73c0700 x1877759534799744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773989.545898:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534799744/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773989.545906:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773989.545910:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773989.545913:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534799744:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773989.545920:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773986/real 1790773989] req@ffff9c4ef73c2a00 x1877759534799872/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773989.545932:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534799872/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773989.545942:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773989.545944:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773989.545948:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534799872:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773989.545955:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773986/real 1790773989] req@ffff9c4ef73c2d80 x1877759534800000/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773989.545981:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534800000/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773989.545994:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773989.545998:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773989.546002:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534800000:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773989.546009:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773986/real 1790773989] req@ffff9c4ef73c3800 x1877759534800128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773989.546021:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534800128/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773989.546036:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773989.546039:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773989.546042:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534800128:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773989.546049:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773986/real 1790773989] req@ffff9c4ef73c0a80 x1877759534800256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773989.546060:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534800256/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773989.546068:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773989.546072:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773989.546075:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534800256:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773989.546082:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773986/real 1790773989] req@ffff9c4ef73c0e00 x1877759534800384/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773989.546094:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534800384/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773989.546104:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773989.546107:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773989.546111:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534800384:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773989.546117:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773986/real 1790773989] req@ffff9c4ef73c2300 x1877759534800512/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773989.546129:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534800512/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773989.546138:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773989.546142:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773989.546145:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534800512:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773989.608779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773989.608779:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773989.609052:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773989.609127:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773989.672784:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773989.673616:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773989.673755:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790773989.800798:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773989.800917:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773989.864780:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773989.928899:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773989.992761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773989.992860:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773989.992936:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773990.056817:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773990.120823:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773990.120826:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773990.120870:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773990.248734:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773990.248736:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773990.249044:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773990.249841:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773990.312790:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773990.312833:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773990.313470:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773990.315688:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773990.315701:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773990.316741:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773990.440967:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773990.504945:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773990.575969:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773990.632779:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773990.632780:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773990.632780:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773990.633030:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773990.696789:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773990.696852:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790773990.698112:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773990.824742:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773990.824755:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773990.888896:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773990.953167:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773991.016807:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773991.016807:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773991.016907:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773991.080776:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790773991.144948:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773991.145152:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773991.145224:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773991.272800:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773991.272800:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773991.272826:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773991.272834:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773991.337187:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773991.337319:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773991.337450:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773991.337619:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773991.337670:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773991.337857:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773991.464790:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773991.528889:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773991.593014:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790773991.593835:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773991.593842:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790773991.593846:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773991.593856:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2987 00000100:00080000:2.0:1790773991.593865:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773991.593877:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790773991.593913:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0380 x1877759534800640/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773991.593934:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773991.593938:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773991.593939:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773991.593944:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2987 00000100:00080000:2.0:1790773991.593948:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773991.593953:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773991.593962:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773991.593975:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2300 x1877759534800768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773991.593985:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773991.593989:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773991.593990:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773991.593995:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2987 00000100:00080000:2.0:1790773991.593998:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773991.594003:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773991.594007:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773991.594018:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1f80 x1877759534800896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773991.594027:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773991.594030:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773991.594033:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773991.594036:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2987 00000100:00080000:2.0:1790773991.594039:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773991.594043:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773991.594048:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790773991.594058:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1c00 x1877759534801024/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773991.594067:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773991.594070:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773991.594072:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773991.594075:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2987 00000100:00080000:2.0:1790773991.594079:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773991.594083:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773991.594087:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790773991.594095:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534801152/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773991.594104:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773991.594107:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790773991.594110:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773991.594114:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2987 00000100:00080000:2.0:1790773991.594117:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773991.594122:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773991.594126:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773991.594136:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534801280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773991.594146:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773991.594149:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790773991.594151:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773991.594156:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2987 00000100:00080000:2.0:1790773991.594159:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773991.594164:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773991.594169:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790773991.594178:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534801408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773991.594187:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773991.594190:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790773991.594192:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773991.594196:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2987 00000100:00080000:2.0:1790773991.594200:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773991.594204:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773991.594208:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790773991.594219:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534801536/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790773991.594228:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790773991.594232:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790773991.594234:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790773991.594238:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2987 00000100:00080000:2.0:1790773991.594241:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790773991.594246:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790773991.594251:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790773991.594260:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534801664/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790773991.594352:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534800640:192.168.204.150@tcp:250:kworker.0 00000800:00000100:1.0:1790773991.594413:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773991.594452:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773991/real 1790773991] req@ffff9c4efbcc0380 x1877759534800640/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773991.594469:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0380 x1877759534800640/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773991.594479:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773991.594483:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773991.594487:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534800640:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790773991.594494:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534800768:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773991.594514:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773991.594525:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773991/real 1790773991] req@ffff9c4efbcc2300 x1877759534800768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773991.594536:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2300 x1877759534800768/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773991.594552:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773991.594556:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773991.594559:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534800768:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773991.594566:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534800896:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773991.594586:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773991.594597:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773991/real 1790773991] req@ffff9c4efbcc1f80 x1877759534800896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773991.594609:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1f80 x1877759534800896/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773991.594618:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773991.594622:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773991.594625:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534800896:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773991.594631:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534801024:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773991.594672:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773991.594684:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773991/real 1790773991] req@ffff9c4efbcc1c00 x1877759534801024/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773991.594697:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1c00 x1877759534801024/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773991.594707:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773991.594711:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773991.594714:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534801024:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773991.594720:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534801152:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773991.594739:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773991.594749:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773991/real 1790773991] req@ffff9c4efbcc3100 x1877759534801152/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773991.594775:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534801152/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773991.594784:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773991.594787:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773991.594790:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534801152:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773991.594796:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534801280:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773991.594814:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773991.594823:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773991/real 1790773991] req@ffff9c4efbcc2680 x1877759534801280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773991.594833:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534801280/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773991.594848:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773991.594851:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773991.594855:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534801280:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773991.594860:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534801408:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790773991.594879:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773991.594889:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773991/real 1790773991] req@ffff9c4efbcc1880 x1877759534801408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773991.594902:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534801408/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773991.594915:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773991.594919:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773991.594922:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534801408:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790773991.594928:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534801536:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773991.594947:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773991.594958:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773991/real 1790773991] req@ffff9c4efbcc1500 x1877759534801536/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773991.594970:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534801536/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773991.594982:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773991.594985:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773991.594989:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534801536:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773991.594994:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534801664:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790773991.595013:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790773991.595022:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773991/real 1790773991] req@ffff9c4efbcc0700 x1877759534801664/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790773991.595033:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534801664/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790773991.595042:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790773991.595045:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790773991.595049:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534801664:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790773991.656765:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773991.656765:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773991.656884:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773991.657265:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773991.720885:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773991.720971:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790773991.721362:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773991.848819:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773991.848976:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773991.912868:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773991.976793:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773992.040803:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773992.040803:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773992.041294:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773992.104839:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773992.168798:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773992.168798:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773992.168821:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773992.296794:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773992.296801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773992.296908:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773992.297041:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773992.361682:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773992.362882:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773992.364310:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773992.364368:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773992.364416:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773992.364640:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773992.489631:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773992.552773:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773992.616794:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773992.680796:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773992.680944:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773992.681142:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773992.681537:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773992.744782:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773992.745301:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773992.745816:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773992.872818:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773992.872886:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773992.936795:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773993.000782:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773993.064781:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773993.064792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773993.065222:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773993.128789:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773993.192792:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773993.192792:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773993.193281:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773993.320780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790773993.320780:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773993.320780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773993.321218:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773993.384942:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790773993.384962:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773993.384981:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773993.385175:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773993.385223:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773993.385825:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773993.512784:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773993.576996:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773993.640797:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773993.704786:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773993.704836:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773993.705341:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773993.705552:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773993.768806:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773993.768904:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:1.0:1790773993.769023:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773993.896800:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773993.896848:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773993.960968:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773994.025228:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773994.088768:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773994.088783:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773994.088978:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773994.152794:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773994.217308:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773994.217484:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773994.217555:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773994.344765:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773994.344765:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773994.344781:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773994.347065:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773994.412761:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773994.412885:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773994.412948:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773994.412980:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773994.419523:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773994.419756:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773994.536798:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773994.600717:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773994.665289:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773994.728789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773994.728886:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773994.728992:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773994.729065:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773994.792780:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773994.792825:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:3.0:1790773994.793363:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773994.920859:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773994.921321:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773994.984892:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773995.049244:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773995.112783:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773995.112998:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773995.113152:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773995.177011:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790773995.240786:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773995.240914:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773995.241341:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773995.368788:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773995.368799:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773995.368891:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773995.369057:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773995.432761:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773995.432813:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773995.432816:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773995.432910:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773995.432932:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773995.432977:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773995.560786:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773995.624960:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790773995.688779:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773995.752759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773995.752759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773995.753089:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773995.753390:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773995.816780:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773995.816790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:3.0:1790773995.817452:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790773995.944774:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773995.944774:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773996.008798:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773996.075114:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773996.136804:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773996.136805:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773996.137162:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773996.200778:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773996.264781:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773996.264782:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773996.264886:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773996.392857:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773996.392857:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773996.392857:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773996.392898:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773996.456747:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773996.456943:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773996.457299:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773996.457950:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773996.458090:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773996.458179:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773996.587182:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773996.650168:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:1.0:1790773996.712987:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773996.712996:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1790773996.713001:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773996.713011:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2992 00000100:00080000:1.0:1790773996.713021:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773996.713032:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:1.0:1790773996.713071:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4df80 x1877759534801792/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773996.713165:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773996.713170:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773996.713172:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773996.713178:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2992 00000100:00080000:1.0:1790773996.713183:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773996.713189:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773996.713198:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773996.713213:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ea00 x1877759534801920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773996.713224:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773996.713227:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773996.713229:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773996.713233:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2992 00000100:00080000:1.0:1790773996.713236:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773996.713241:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773996.713245:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773996.713255:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4fb80 x1877759534802048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773996.713274:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773996.713276:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773996.713278:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773996.713282:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2992 00000100:00080000:1.0:1790773996.713286:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773996.713290:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773996.713295:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:1.0:1790773996.713305:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4dc00 x1877759534802176/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773996.713313:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773996.713316:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773996.713318:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773996.713322:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2992 00000100:00080000:1.0:1790773996.713325:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773996.713329:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773996.713335:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:1.0:1790773996.713344:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4e680 x1877759534802304/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773996.713352:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773996.713356:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1790773996.713358:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773996.713361:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2992 00000100:00080000:1.0:1790773996.713364:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773996.713369:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773996.713373:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773996.713383:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f800 x1877759534802432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773996.713392:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773996.713395:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1790773996.713397:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773996.713401:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2992 00000100:00080000:1.0:1790773996.713404:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773996.713409:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773996.713419:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1790773996.713434:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4f480 x1877759534802560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773996.713444:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773996.713447:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1790773996.713449:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773996.713453:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2992 00000100:00080000:1.0:1790773996.713457:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773996.713463:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773996.713467:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:1.0:1790773996.713476:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4ed80 x1877759534802688/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1790773996.713485:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1790773996.713488:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1790773996.713490:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1790773996.713494:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2992 00000100:00080000:1.0:1790773996.713497:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1790773996.713501:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:1.0:1790773996.713506:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:1.0:1790773996.713516:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efac4c700 x1877759534802816/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1790773996.713570:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773996.713640:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534801792:192.168.204.150@tcp:250:kworker.0 00000800:00000100:2.0:1790773996.713729:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773996.713767:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773996/real 1790773996] req@ffff9c4efac4df80 x1877759534801792/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773996.713783:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4df80 x1877759534801792/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773996.713792:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773996.713796:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773996.713800:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534801792:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790773996.713806:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534801920:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773996.713824:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773996.713836:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773996/real 1790773996] req@ffff9c4efac4ea00 x1877759534801920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773996.713847:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ea00 x1877759534801920/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773996.713862:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773996.713865:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773996.713869:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534801920:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773996.713873:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534802048:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773996.713891:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773996.713901:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773996/real 1790773996] req@ffff9c4efac4fb80 x1877759534802048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773996.713912:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4fb80 x1877759534802048/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773996.713921:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773996.713925:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773996.713928:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534802048:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773996.713934:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534802176:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773996.713951:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773996.713961:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773996/real 1790773996] req@ffff9c4efac4dc00 x1877759534802176/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773996.713972:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4dc00 x1877759534802176/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773996.713982:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773996.713985:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773996.713988:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534802176:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773996.713993:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534802304:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773996.714007:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773996.714016:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773996/real 1790773996] req@ffff9c4efac4e680 x1877759534802304/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773996.714025:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4e680 x1877759534802304/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773996.714035:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773996.714038:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773996.714040:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534802304:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773996.714045:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534802432:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773996.714060:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773996.714069:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773996/real 1790773996] req@ffff9c4efac4f800 x1877759534802432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773996.714080:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f800 x1877759534802432/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773996.714096:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773996.714099:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773996.715188:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534802432:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773996.715199:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534802560:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790773996.715228:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773996.715243:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773996/real 1790773996] req@ffff9c4efac4f480 x1877759534802560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773996.715263:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4f480 x1877759534802560/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773996.715272:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773996.715275:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773996.715279:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534802560:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790773996.715285:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534802688:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773996.715302:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773996.715311:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773996/real 1790773996] req@ffff9c4efac4ed80 x1877759534802688/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773996.715322:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4ed80 x1877759534802688/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773996.715333:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773996.715336:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773996.715340:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534802688:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790773996.715346:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534802816:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790773996.715372:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790773996.715382:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790773996/real 1790773996] req@ffff9c4efac4c700 x1877759534802816/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790773996.715393:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efac4c700 x1877759534802816/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790773996.715407:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790773996.715410:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790773996.715414:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efac4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534802816:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790773996.776747:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773996.776747:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773996.776790:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773996.776819:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773996.840868:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790773996.841041:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773996.841629:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790773996.968790:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773996.968931:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773997.032781:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773997.097014:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773997.160786:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790773997.160939:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773997.163222:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773997.225108:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773997.288800:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773997.289693:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773997.289837:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773997.416799:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773997.416853:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773997.417220:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790773997.417814:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773997.480759:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773997.480768:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790773997.481164:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773997.481566:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773997.482127:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790773997.482192:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773997.609699:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773997.673309:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773997.738047:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773997.800774:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790773997.800789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773997.800804:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773997.804454:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790773997.864773:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773997.865022:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773997.865089:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773997.992784:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773997.994096:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773998.056783:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773998.120773:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773998.184791:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773998.184920:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790773998.185023:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773998.249094:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790773998.312785:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773998.312828:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773998.313341:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773998.440766:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790773998.440766:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773998.440999:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773998.441202:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773998.504773:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773998.505284:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773998.505414:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790773998.506032:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773998.506066:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773998.506162:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790773998.632807:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773998.696787:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773998.760760:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790773998.824915:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773998.825085:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790773998.825188:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773998.825214:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773998.888731:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790773998.888880:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790773998.888937:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790773999.017752:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790773999.021126:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790773999.080922:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773999.147945:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790773999.208788:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773999.210561:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790773999.212934:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773999.272784:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790773999.336793:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773999.336942:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790773999.337061:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790773999.464868:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773999.465098:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790773999.465779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790773999.465837:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790773999.528764:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790773999.528764:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790773999.528858:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790773999.528910:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790773999.529007:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790773999.529043:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790773999.656756:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790773999.722263:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790773999.784773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773999.848783:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790773999.848902:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790773999.849023:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790773999.849331:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790773999.912801:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790773999.912935:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790773999.913081:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774000.040786:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774000.040974:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774000.105329:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774000.168790:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774000.232765:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790774000.232765:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774000.233036:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774000.296766:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790774000.360975:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774000.361079:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774000.362159:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774000.488957:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790774000.489056:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790774000.489533:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774000.489986:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774000.552781:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774000.552781:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774000.552911:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790774000.552937:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790774000.552949:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790774000.552982:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790774000.680792:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790774000.744994:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774000.809055:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:2.0:1790774000.872813:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790774000.872845:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790774000.872951:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774000.873127:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774000.936813:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:3.0:1790774000.937272:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790774000.937308:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790774001.064790:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774001.065280:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774001.128798:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774001.192818:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774001.256771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774001.256797:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790774001.257636:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774001.320815:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790774001.384781:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774001.384781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774001.385610:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774001.512798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790774001.513061:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774001.513075:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774001.513140:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790774001.576774:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790774001.576826:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774001.576881:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790774001.577049:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790774001.577095:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774001.578399:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774001.704856:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:3.0:1790774001.768781:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774001.832887:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790774001.833797:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774001.833805:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790774001.833809:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774001.833818:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 2997 00000100:00080000:3.0:1790774001.833828:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774001.833839:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790774001.833877:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534802944/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774001.833907:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774001.833911:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790774001.833913:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774001.833917:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2997 00000100:00080000:3.0:1790774001.833920:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774001.833924:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774001.833933:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790774001.833945:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534803072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774001.833954:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774001.833958:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790774001.833959:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774001.833963:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2997 00000100:00080000:3.0:1790774001.833966:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774001.833969:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774001.833973:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790774001.833980:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534803200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774001.833989:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774001.833992:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790774001.833993:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774001.833997:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2997 00000100:00080000:3.0:1790774001.834000:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774001.834003:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774001.834007:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790774001.834013:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534803328/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774001.834021:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774001.834024:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790774001.834026:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774001.834030:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 2997 00000100:00080000:3.0:1790774001.834033:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774001.834037:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774001.834041:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790774001.834048:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534803456/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774001.834055:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774001.834059:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790774001.834060:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774001.834064:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2997 00000100:00080000:3.0:1790774001.834066:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774001.834070:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774001.834074:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790774001.834082:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534803584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774001.834091:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774001.834095:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790774001.834097:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774001.834101:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2997 00000100:00080000:3.0:1790774001.834104:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774001.834108:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774001.834112:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790774001.834120:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534803712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774001.834128:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774001.834131:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790774001.834133:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774001.834136:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2997 00000100:00080000:3.0:1790774001.834139:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774001.834143:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774001.834146:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790774001.834156:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534803840/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774001.834164:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774001.834167:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790774001.834168:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774001.834171:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 2997 00000100:00080000:3.0:1790774001.834174:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774001.834177:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774001.834180:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790774001.834188:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534803968/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790774001.834265:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534802944:192.168.204.150@tcp:250:kworker.0 00000800:00000100:2.0:1790774001.834327:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774001.834364:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774001/real 1790774001] req@ffff9c4ef73c1c00 x1877759534802944/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774001.834379:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534802944/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774001.834388:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774001.834392:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774001.834396:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534802944:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790774001.834406:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534803072:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790774001.834422:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774001.834431:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774001/real 1790774001] req@ffff9c4ef73c2300 x1877759534803072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774001.834442:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534803072/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774001.834457:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774001.834460:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774001.834463:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534803072:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790774001.834468:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534803200:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790774001.834481:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774001.834489:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774001/real 1790774001] req@ffff9c4ef73c0e00 x1877759534803200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774001.834499:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534803200/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774001.834507:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774001.834510:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774001.834513:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534803200:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790774001.834518:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534803328:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790774001.834531:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774001.834539:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774001/real 1790774001] req@ffff9c4ef73c0a80 x1877759534803328/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774001.834550:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534803328/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774001.834573:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774001.834576:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774001.834579:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534803328:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790774001.834584:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534803456:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790774001.834600:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774001.834609:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774001/real 1790774001] req@ffff9c4ef73c3800 x1877759534803456/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774001.834620:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534803456/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774001.834629:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774001.834632:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774001.834635:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534803456:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790774001.834640:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534803584:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790774001.834674:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774001.834683:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774001/real 1790774001] req@ffff9c4ef73c2d80 x1877759534803584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774001.835853:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534803584/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774001.835877:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774001.835881:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774001.835885:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534803584:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790774001.835893:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534803712:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790774001.835930:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774001.835945:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774001/real 1790774001] req@ffff9c4ef73c2a00 x1877759534803712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774001.835956:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534803712/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774001.835965:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774001.835968:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774001.835971:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534803712:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790774001.835978:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534803840:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790774001.835997:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774001.836008:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774001/real 1790774001] req@ffff9c4ef73c0700 x1877759534803840/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774001.836020:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534803840/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774001.836030:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774001.836033:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774001.836037:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534803840:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790774001.836043:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534803968:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790774001.836062:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774001.836072:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774001/real 1790774001] req@ffff9c4ef73c1500 x1877759534803968/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774001.836084:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534803968/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774001.836093:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774001.836097:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774001.836100:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534803968:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790774001.896765:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790774001.896771:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790774001.896829:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774001.896909:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774001.960758:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774001.960761:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790774001.960881:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790774002.088942:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774002.088942:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774002.152814:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774002.219129:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774002.280792:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774002.280842:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774002.281193:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790774002.344776:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790774002.408824:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774002.408824:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774002.408960:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774002.536810:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774002.536810:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774002.536830:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790774002.536911:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790774002.600892:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774002.601004:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790774002.601106:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774002.601354:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790774002.601442:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790774002.601510:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790774002.728791:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774002.795399:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774002.856870:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790774002.920837:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774002.920879:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774002.920893:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774002.921133:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790774002.985962:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790774002.985981:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790774002.986840:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790774003.112781:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774003.113229:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774003.177969:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774003.240834:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774003.304812:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790774003.305112:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774003.308328:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774003.368805:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790774003.432775:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774003.432866:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774003.433490:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774003.560817:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774003.560982:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774003.561220:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774003.562238:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774003.625128:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790774003.625283:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790774003.625532:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774003.625850:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790774003.626485:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774003.626607:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774003.752884:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774003.818824:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774003.880770:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790774003.944755:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774003.944759:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774003.944793:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790774003.944808:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790774004.008801:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790774004.009436:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790774004.009483:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:0.0:1790774004.136793:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774004.137069:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774004.200773:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774004.265262:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774004.328776:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790774004.329439:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774004.329581:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774004.393494:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790774004.456778:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774004.456790:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774004.456974:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774004.584779:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790774004.584779:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790774004.584987:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774004.585673:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790774004.648787:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774004.648787:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774004.648930:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790774004.648943:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774004.651002:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790774004.651007:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774004.776844:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774004.840880:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774004.905466:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774004.968877:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790774004.969810:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774004.969925:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774004.970127:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790774005.032819:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790774005.032903:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:2.0:1790774005.033368:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790774005.160735:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774005.160736:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774005.225219:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774005.288762:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774005.352799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790774005.352809:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774005.359342:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774005.416769:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790774005.480813:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774005.480816:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774005.480821:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774005.608794:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774005.608800:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774005.608801:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774005.608902:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790774005.672762:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790774005.672762:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790774005.672869:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774005.672938:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774005.672983:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790774005.673034:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790774005.801326:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774005.864745:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774005.928755:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774005.992766:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790774005.992767:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774005.993035:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774005.993035:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790774006.057271:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774006.057414:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790774006.057735:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790774006.184764:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774006.185288:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774006.248808:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774006.312744:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774006.376804:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774006.377383:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774006.377458:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790774006.440782:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790774006.504749:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774006.504750:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774006.504750:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774006.632780:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774006.632780:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774006.632843:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774006.633095:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790774006.696871:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774006.696928:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790774006.697638:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790774006.697744:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790774006.697899:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790774006.698065:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774006.826872:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774006.888821:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774006.952787:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:2.0:1790774006.956673:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790774006.956683:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1790774006.956689:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790774006.958355:0:18871:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 3002 00000100:00080000:2.0:1790774006.958369:0:18871:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790774006.958383:0:18871:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:2.0:1790774006.958425:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534804096/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790774006.958475:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790774006.958479:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790774006.958482:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790774006.958487:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 3002 00000100:00080000:2.0:1790774006.958491:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790774006.958496:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790774006.958506:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790774006.958520:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534804224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790774006.958528:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534804096:192.168.204.150@tcp:250:kworker.0 00000100:00080000:2.0:1790774006.958535:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1790774006.958595:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790774006.958634:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774006/real 1790774006] req@ffff9c4efbcc0a80 x1877759534804096/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790774006.958685:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534804096/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790774006.958697:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790774006.958701:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790774006.958705:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534804096:192.168.204.150@tcp:250:kworker.0 00000100:00100000:1.0:1790774006.958731:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534804224:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790774006.958754:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790774006.958766:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774006/real 1790774006] req@ffff9c4efbcc2a00 x1877759534804224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790774006.958778:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534804224/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790774006.958796:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790774006.958800:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790774006.958804:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534804224:192.168.204.150@tcp:38:kworker.0 00000100:00080000:2.0:1790774006.960113:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790774006.960119:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790774006.960131:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 3002 00000100:00080000:2.0:1790774006.960141:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790774006.960153:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790774006.960168:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790774006.960204:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2a00 x1877759534804352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790774006.960234:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790774006.960239:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790774006.960242:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790774006.960246:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 3002 00000100:00080000:2.0:1790774006.960250:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790774006.960255:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790774006.960260:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:2.0:1790774006.960271:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0a80 x1877759534804480/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790774006.960281:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790774006.960285:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790774006.960287:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790774006.960291:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 3002 00000100:00080000:2.0:1790774006.960294:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790774006.960299:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790774006.960303:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:2.0:1790774006.960312:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc0700 x1877759534804608/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790774006.960321:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790774006.960325:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1790774006.960327:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790774006.960331:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 3002 00000100:00080000:2.0:1790774006.960334:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790774006.960339:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790774006.960343:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790774006.960353:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1500 x1877759534804736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790774006.960367:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790774006.960370:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1790774006.960372:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790774006.960377:0:18871:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 3002 00000100:00080000:2.0:1790774006.960380:0:18871:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790774006.960385:0:18871:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790774006.960389:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1790774006.960398:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc1880 x1877759534804864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790774006.960407:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790774006.960422:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1790774006.960424:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790774006.960429:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 3002 00000100:00080000:2.0:1790774006.960432:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790774006.960437:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790774006.960442:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:2.0:1790774006.960465:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc2680 x1877759534804992/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1790774006.960476:0:18871:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1790774006.960479:0:18871:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1790774006.960481:0:18871:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1790774006.960486:0:18871:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 3002 00000100:00080000:2.0:1790774006.960489:0:18871:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1790774006.960494:0:18871:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:2.0:1790774006.960500:0:18871:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:2.0:1790774006.960509:0:18871:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4efbcc3100 x1877759534805120/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1790774006.961931:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534804352:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790774006.962017:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790774006.962045:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774006/real 1790774006] req@ffff9c4efbcc2a00 x1877759534804352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790774006.962069:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2a00 x1877759534804352/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790774006.962081:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790774006.962085:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790774006.962090:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534804352:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790774006.962097:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534804480:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790774006.962119:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790774006.962131:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774006/real 1790774006] req@ffff9c4efbcc0a80 x1877759534804480/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790774006.962142:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0a80 x1877759534804480/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790774006.962166:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790774006.962170:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790774006.962173:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534804480:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790774006.962180:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534804608:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790774006.962198:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790774006.962208:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774006/real 1790774006] req@ffff9c4efbcc0700 x1877759534804608/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790774006.962218:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc0700 x1877759534804608/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790774006.962232:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790774006.962235:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790774006.962238:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534804608:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790774006.962244:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534804736:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790774006.962261:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790774006.962280:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774006/real 1790774006] req@ffff9c4efbcc1500 x1877759534804736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790774006.962292:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1500 x1877759534804736/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790774006.962309:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790774006.962312:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790774006.962315:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534804736:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790774006.962321:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534804864:192.168.204.150@tcp:38:kworker.0 00000800:00000100:1.0:1790774006.962339:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790774006.962348:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774006/real 1790774006] req@ffff9c4efbcc1880 x1877759534804864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790774006.962358:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc1880 x1877759534804864/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790774006.962365:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790774006.962368:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790774006.962371:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534804864:192.168.204.150@tcp:38:kworker.0 00000100:00100000:1.0:1790774006.962377:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534804992:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790774006.962393:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790774006.962401:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774006/real 1790774006] req@ffff9c4efbcc2680 x1877759534804992/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790774006.962411:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc2680 x1877759534804992/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790774006.962420:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790774006.962423:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790774006.962426:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534804992:192.168.204.150@tcp:8:kworker.0 00000100:00100000:1.0:1790774006.962432:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534805120:192.168.204.150@tcp:8:kworker.0 00000800:00000100:1.0:1790774006.962446:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:1.0:1790774006.962467:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774006/real 1790774006] req@ffff9c4efbcc3100 x1877759534805120/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1790774006.962478:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4efbcc3100 x1877759534805120/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1790774006.962487:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1790774006.962490:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:1.0:1790774006.962493:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4efbcc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534805120:192.168.204.150@tcp:8:kworker.0 00000100:00100000:0.0:1790774007.016789:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774007.016910:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774007.017062:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790774007.018676:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790774007.082030:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790774007.082223:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790774007.082300:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774007.208803:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774007.208803:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774007.272928:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774007.336798:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774007.400781:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774007.400905:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790774007.400978:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774007.464773:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790774007.528781:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774007.528782:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774007.528987:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774007.656763:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790774007.656989:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790774007.657066:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774007.657489:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774007.720782:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790774007.720799:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774007.720807:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790774007.720810:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790774007.720830:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774007.720845:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774007.848757:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774007.912893:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774007.976804:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774008.040818:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790774008.040818:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790774008.040821:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774008.043401:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00010000:00010000:0.0:1790774008.105005:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790774008.105005:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774008.105006:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790774008.232836:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774008.232957:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774008.297161:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774008.361005:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774008.424806:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790774008.424848:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774008.425501:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774008.488806:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:3.0:1790774008.552793:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774008.552942:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774008.553050:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774008.680757:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790774008.680757:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774008.680757:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790774008.680758:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774008.744781:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790774008.744798:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790774008.744913:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790774008.745296:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774008.745720:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790774008.745817:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774008.872799:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774008.936755:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774009.000830:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790774009.064773:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774009.064784:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774009.064784:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790774009.064785:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774009.128826:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790774009.128858:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790774009.129045:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790774009.256798:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774009.256801:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774009.322823:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774009.384818:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774009.448827:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790774009.448827:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774009.449506:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774009.515092:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790774009.576814:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774009.576840:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774009.576955:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774009.704827:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790774009.704878:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774009.704909:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790774009.704922:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790774009.768917:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790774009.768929:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774009.769910:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790774009.770045:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774009.770165:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774009.770777:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774009.896884:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774009.960778:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774010.024965:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774010.088768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774010.088768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774010.088768:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790774010.088925:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790774010.152918:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774010.153764:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790774010.153813:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790774010.280805:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774010.281205:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774010.345238:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774010.408867:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774010.472799:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790774010.472799:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774010.473115:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774010.537432:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790774010.600781:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774010.600906:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774010.601449:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774010.728783:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774010.728957:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774010.729048:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774010.729054:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790774010.792809:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774010.793103:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774010.793433:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790774010.793454:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790774010.793580:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774010.793676:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790774010.920845:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774010.985044:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774011.049614:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774011.112825:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774011.113285:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:3.0:1790774011.113593:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774011.113670:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790774011.176853:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790774011.177335:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790774011.177543:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790774011.304769:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774011.304818:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774011.369492:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774011.432899:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774011.496717:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774011.496740:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790774011.496770:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774011.560820:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790774011.624768:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774011.624768:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774011.624984:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774011.752783:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790774011.752885:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774011.753000:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774011.753054:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:0.0:1790774011.816761:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1790774011.816761:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774011.816787:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790774011.816881:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774011.816900:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774011.816946:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790774011.944805:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774012.008985:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774012.072835:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790774012.076957:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774012.076967:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790774012.076972:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774012.076982:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 3007 00000100:00080000:3.0:1790774012.076993:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774012.077004:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790774012.077042:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0000 x1877759534805248/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774012.077071:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774012.077075:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790774012.077077:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774012.077081:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 3007 00000100:00080000:3.0:1790774012.077085:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774012.077091:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774012.077100:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790774012.077114:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534805376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774012.077124:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774012.077128:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790774012.077130:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774012.077134:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 3007 00000100:00080000:3.0:1790774012.077138:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774012.077143:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774012.077147:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790774012.077158:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534805504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774012.077167:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774012.077170:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790774012.077172:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774012.077175:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 3007 00000100:00080000:3.0:1790774012.077178:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774012.077183:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774012.077187:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790774012.077196:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534805632/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774012.077205:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774012.077209:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790774012.077211:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774012.077215:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 3007 00000100:00080000:3.0:1790774012.077219:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774012.077223:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774012.077227:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790774012.077238:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534805760/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774012.077247:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774012.077250:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790774012.077252:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774012.077256:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 3007 00000100:00080000:3.0:1790774012.077259:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774012.077263:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774012.077268:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790774012.077277:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534805888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774012.077285:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774012.077289:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790774012.077290:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774012.077294:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 3007 00000100:00080000:3.0:1790774012.077297:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774012.077301:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774012.077306:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790774012.077315:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534806016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774012.077323:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774012.077326:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790774012.077327:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774012.077332:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 3007 00000100:00080000:3.0:1790774012.077335:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774012.077339:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774012.077343:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790774012.077362:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534806144/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774012.077371:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774012.077375:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790774012.077376:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774012.077381:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 3007 00000100:00080000:3.0:1790774012.077384:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774012.077389:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774012.077397:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790774012.077412:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534806272/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790774012.077520:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534805248:192.168.204.150@tcp:250:kworker.0 00000800:00000100:2.0:1790774012.077634:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774012.077692:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774012/real 1790774012] req@ffff9c4ef73c0000 x1877759534805248/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774012.077710:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0000 x1877759534805248/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774012.077721:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774012.077726:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774012.077730:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534805248:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790774012.077737:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534805376:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790774012.077761:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774012.077774:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774012/real 1790774012] req@ffff9c4ef73c1500 x1877759534805376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774012.077786:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534805376/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774012.077802:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774012.077805:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774012.077809:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534805376:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790774012.077815:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534805504:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790774012.077835:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774012.077846:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774012/real 1790774012] req@ffff9c4ef73c0700 x1877759534805504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774012.077857:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534805504/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774012.077864:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774012.077868:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774012.077871:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534805504:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790774012.077877:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534805632:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790774012.077894:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774012.077902:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774012/real 1790774012] req@ffff9c4ef73c2a00 x1877759534805632/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774012.077913:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534805632/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774012.077923:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774012.077926:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774012.077930:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534805632:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790774012.077936:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534805760:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790774012.077952:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774012.077961:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774012/real 1790774012] req@ffff9c4ef73c2d80 x1877759534805760/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774012.077973:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534805760/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774012.077983:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774012.077987:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774012.077989:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534805760:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790774012.077995:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534805888:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790774012.078011:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774012.078020:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774012/real 1790774012] req@ffff9c4ef73c3800 x1877759534805888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774012.078154:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534805888/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774012.078169:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774012.078172:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774012.078177:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534805888:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790774012.078183:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534806016:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790774012.078205:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774012.078217:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774012/real 1790774012] req@ffff9c4ef73c0a80 x1877759534806016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774012.078228:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534806016/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774012.078236:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774012.078240:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774012.078243:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534806016:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790774012.078249:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534806144:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790774012.078269:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774012.078280:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774012/real 1790774012] req@ffff9c4ef73c0e00 x1877759534806144/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774012.078292:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534806144/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774012.078302:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774012.078305:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774012.078309:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534806144:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790774012.078316:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534806272:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790774012.078335:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774012.078346:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774012/real 1790774012] req@ffff9c4ef73c2300 x1877759534806272/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774012.078375:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534806272/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774012.078385:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774012.078389:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774012.078392:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534806272:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790774012.136774:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774012.136812:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774012.138278:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774012.138447:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790774012.200778:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:1.0:1790774012.200829:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790774012.201224:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790774012.328775:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774012.328914:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774012.394302:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774012.456908:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774012.520761:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774012.520778:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790774012.520866:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774012.584751:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790774012.648926:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774012.650765:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774012.650810:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774012.776777:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774012.776795:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774012.776938:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790774012.776952:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774012.840808:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774012.840815:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774012.840849:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790774012.840875:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774012.841180:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790774012.841199:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790774012.968833:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774013.032933:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774013.096757:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774013.160768:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790774013.160768:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774013.160769:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790774013.160880:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790774013.224774:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790774013.224941:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790774013.225010:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790774013.352781:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774013.352782:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774013.416824:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774013.480799:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774013.544782:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774013.544783:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774013.544807:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790774013.608783:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790774013.672762:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774013.672762:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774013.672762:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774013.800792:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790774013.800793:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774013.800930:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774013.800983:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774013.864777:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:3.0:1790774013.864808:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790774013.864810:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790774013.864811:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774013.864841:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774013.864842:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774013.992788:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774014.057012:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:0.0:1790774014.120903:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774014.184820:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790774014.184820:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774014.184855:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774014.185053:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:2.0:1790774014.248857:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790774014.248934:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790774014.249026:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774014.376805:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774014.376958:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774014.441960:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774014.504778:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774014.568797:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774014.568924:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790774014.569848:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774014.632842:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790774014.696827:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774014.699160:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774014.699984:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774014.825703:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:3.0:1790774014.825720:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774014.826442:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774014.826459:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790774014.888830:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790774014.888838:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774014.888863:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774014.888894:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:1.0:1790774014.888914:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790774014.888927:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774015.016889:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774015.080765:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790774015.145113:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:3.0:1790774015.208727:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774015.208998:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790774015.208998:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774015.209000:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774015.272829:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774015.272881:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:1.0:1790774015.272970:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790774015.400777:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774015.401571:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774015.470712:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774015.529444:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774015.592770:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774015.592771:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:1.0:1790774015.592771:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774015.656764:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:2.0:1790774015.720759:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774015.720759:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774015.720832:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774015.848776:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:1.0:1790774015.851427:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790774015.854607:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774015.856798:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774015.912785:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790774015.912785:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774015.912906:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774015.912957:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790774015.913119:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:2.0:1790774015.913175:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774016.040985:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774016.104764:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790774016.168832:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774016.232819:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790774016.233757:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774016.233776:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790774016.236460:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00010000:00010000:1.0:1790774016.296860:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790774016.296968:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790774016.297167:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774016.425129:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774016.425307:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774016.489161:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774016.552771:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774016.616750:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774016.616819:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774016.616900:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:0.0:1790774016.681498:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790774016.744814:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774016.744884:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774016.744934:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774016.872812:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774016.873242:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774016.873379:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774016.873423:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:0.0:1790774016.936791:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774016.936791:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774016.936793:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1790774016.936822:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774016.936886:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790774016.936900:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790774017.064792:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774017.128766:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00080000:3.0:1790774017.192794:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) b1889553-821d-4c14-8f1b-bf5aa6dae342->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774017.192803:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1790774017.192808:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1790774017.192809:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00080000:3.0:1790774017.192820:0:15335:0:(import.c:545:import_select_connection()) MGC192.168.204.150@tcp: connect to NID 192.168.204.150@tcp last attempt 3013 00000100:00080000:3.0:1790774017.192830:0:15335:0:(import.c:630:import_select_connection()) MGC192.168.204.150@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774017.192841:0:15335:0:(import.c:666:import_select_connection()) MGC192.168.204.150@tcp: import ffff9c4ec5419800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000100:00100000:3.0:1790774017.192875:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1c00 x1877759534806400/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774017.192916:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774017.192920:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790774017.192922:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774017.192926:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 3013 00000100:00080000:3.0:1790774017.192929:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774017.192934:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1f800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774017.192942:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790774017.192955:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2300 x1877759534806528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774017.192964:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774017.192968:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790774017.192969:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774017.192973:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 3013 00000100:00080000:3.0:1790774017.192975:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774017.192979:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ec54bd800: import ffff9c4ec4a1b000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774017.192983:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790774017.193000:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0e00 x1877759534806656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774017.193009:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774017.193012:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790774017.193014:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774017.193018:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 3013 00000100:00080000:3.0:1790774017.193021:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774017.193025:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ec54bd800: import ffff9c4ec4a1b800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774017.193029:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2498560, lost: 0. 00000100:00100000:3.0:1790774017.193037:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0a80 x1877759534806784/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774017.193045:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) dd3e9fa3-001c-49c2-ab75-41a75fa63682->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774017.193048:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790774017.193050:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774017.193053:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: connect to NID 192.168.204.150@tcp last attempt 3013 00000100:00080000:3.0:1790774017.193056:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774017.193060:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ec54bd800: import ffff9c4ec4a18000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774017.193063:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 2289664, lost: 0. 00000100:00100000:3.0:1790774017.193071:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c3800 x1877759534806912/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774017.193078:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774017.193081:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1790774017.193083:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774017.193086:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 3013 00000100:00080000:3.0:1790774017.193089:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774017.193093:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff9c4ee0fe1800: import ffff9c4ee0fe1000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774017.193096:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790774017.193103:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2d80 x1877759534807040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774017.193112:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774017.193115:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1790774017.193117:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774017.193121:0:15335:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 3013 00000100:00080000:3.0:1790774017.193124:0:15335:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774017.193127:0:15335:0:(import.c:666:import_select_connection()) lustre-MDT0001-mdc-ffff9c4ee0fe1800: import ffff9c4ed0cca800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774017.193131:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be37493b8 ocd_version: 34683651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1790774017.193139:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c2a00 x1877759534807168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774017.193147:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774017.193150:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1790774017.193152:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774017.193156:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 3013 00000100:00080000:3.0:1790774017.193168:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774017.193173:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff9c4ee0fe1800: import ffff9c4ed0cc8800 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774017.193177:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 13885440, lost: 0. 00000100:00100000:3.0:1790774017.193187:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c0700 x1877759534807296/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1790774017.193195:0:15335:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82a25f2c-222e-4ba7-abee-8e26bca511e6->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1790774017.193198:0:15335:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1790774017.193200:0:15335:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1790774017.193204:0:15335:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: connect to NID 192.168.204.150@tcp last attempt 3013 00000100:00080000:3.0:1790774017.193207:0:15335:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1790774017.193212:0:15335:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff9c4ee0fe1800: import ffff9c4ed0ccd000 using connection 192.168.204.150@tcp/192.168.204.150@tcp 00000008:00100000:3.0:1790774017.193215:0:15335:0:(osc_request.c:3836:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683651 ocd_grant: 9199616, lost: 0. 00000100:00100000:3.0:1790774017.193225:0:15335:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c4ef73c1500 x1877759534807424/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1790774017.193398:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534806400:192.168.204.150@tcp:250:kworker.0 00000800:00000100:2.0:1790774017.193454:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774017.193491:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774017/real 1790774017] req@ffff9c4ef73c1c00 x1877759534806400/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774017.193507:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1c00 x1877759534806400/t0(0) o250->MGC192.168.204.150@tcp@192.168.204.150@tcp:26/25 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774017.193517:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec5419800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774017.193521:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774017.193525:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b1889553-821d-4c14-8f1b-bf5aa6dae342:2395:1877759534806400:192.168.204.150@tcp:250:kworker.0 00000100:00100000:2.0:1790774017.193532:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534806528:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790774017.193550:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774017.193563:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774017/real 1790774017] req@ffff9c4ef73c2300 x1877759534806528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774017.193574:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2300 x1877759534806528/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774017.193591:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774017.193594:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774017.193597:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534806528:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790774017.193603:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534806656:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790774017.193618:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774017.193626:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774017/real 1790774017] req@ffff9c4ef73c0e00 x1877759534806656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774017.193637:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0e00 x1877759534806656/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ec54bd800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774017.193645:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774017.193648:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774017.193669:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534806656:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790774017.193674:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534806784:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790774017.193751:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774017.193763:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774017/real 1790774017] req@ffff9c4ef73c0a80 x1877759534806784/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774017.193774:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0a80 x1877759534806784/t0(0) o8->lustre-OST0000-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774017.193783:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a1b800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774017.193786:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774017.193790:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534806784:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790774017.193795:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534806912:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790774017.193810:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774017.193819:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774017/real 1790774017] req@ffff9c4ef73c3800 x1877759534806912/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774017.193829:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c3800 x1877759534806912/t0(0) o8->lustre-OST0001-osc-ffff9c4ec54bd800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774017.193838:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ec4a18000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774017.193841:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774017.193844:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:dd3e9fa3-001c-49c2-ab75-41a75fa63682:2395:1877759534806912:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790774017.193849:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534807040:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790774017.193864:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774017.193874:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774017/real 1790774017] req@ffff9c4ef73c2d80 x1877759534807040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774017.193884:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2d80 x1877759534807040/t0(0) o38->lustre-MDT0000-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774017.193899:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ee0fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774017.193902:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774017.193905:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534807040:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790774017.193911:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534807168:192.168.204.150@tcp:38:kworker.0 00000800:00000100:2.0:1790774017.193924:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774017.193934:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774017/real 1790774017] req@ffff9c4ef73c2a00 x1877759534807168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774017.193944:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c2a00 x1877759534807168/t0(0) o38->lustre-MDT0001-mdc-ffff9c4ee0fe1800@192.168.204.150@tcp:12/10 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774017.193953:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cca800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774017.193956:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774017.193959:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534807168:192.168.204.150@tcp:38:kworker.0 00000100:00100000:2.0:1790774017.193964:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534807296:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790774017.193978:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774017.193986:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774017/real 1790774017] req@ffff9c4ef73c0700 x1877759534807296/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774017.193997:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c0700 x1877759534807296/t0(0) o8->lustre-OST0000-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774017.194006:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0cc8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774017.194009:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774017.194012:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534807296:192.168.204.150@tcp:8:kworker.0 00000100:00100000:2.0:1790774017.194017:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534807424:192.168.204.150@tcp:8:kworker.0 00000800:00000100:2.0:1790774017.194029:0:2395:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.150@tcp 00000100:00100000:2.0:1790774017.194037:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1790774017/real 1790774017] req@ffff9c4ef73c1500 x1877759534807424/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1790774017.194047:0:2395:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c4ef73c1500 x1877759534807424/t0(0) o8->lustre-OST0001-osc-ffff9c4ee0fe1800@192.168.204.150@tcp:28/4 lens 520/544 e 0 to 1 dl 1790774042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1790774017.194056:0:2395:0:(import.c:65:import_set_state_nolock()) ffff9c4ed0ccd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1790774017.194059:0:2395:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.150@tcp failed (-110) 00000100:00100000:2.0:1790774017.194063:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff9c4ef73c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82a25f2c-222e-4ba7-abee-8e26bca511e6:2395:1877759534807424:192.168.204.150@tcp:8:kworker.0 00000100:00100000:3.0:1790774017.256767:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774017.256783:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774017.256806:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790774017.256838:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00010000:00010000:1.0:1790774017.320883:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1790774017.320900:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790774017.320910:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774017.448771:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774017.448878:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774017.512768:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774017.576779:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774017.640788:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790774017.640788:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774017.640803:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774017.704910:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790774017.768824:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774017.769628:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774017.769720:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774017.896774:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774017.896774:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774017.896834:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774017.896983:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:0.0:1790774017.960797:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:1.0:1790774017.960797:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:3.0:1790774017.960815:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774017.960936:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790774017.960998:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774017.961453:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774018.089090:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774018.152939:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790774018.216799:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774018.280790:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:0.0:1790774018.280934:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774018.281052:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774018.281570:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774018.344793:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00010000:00010000:0.0:1790774018.345998:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790774018.346083:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:3.0:1790774018.472772:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774018.472873:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774018.536864:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774018.600763:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774018.665320:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790774018.665397:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774018.665557:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774018.729190:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790774018.792736:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774018.792736:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774018.792845:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774018.920760:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790774018.920761:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790774018.920788:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774018.920797:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774018.984774:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:2.0:1790774018.984774:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774018.984781:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774018.984790:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774018.984814:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:1.0:1790774018.984818:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:0.0:1790774019.112771:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774019.176763:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790774019.240773:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774019.304759:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774019.304760:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774019.304933:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:3.0:1790774019.305084:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790774019.368818:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774019.369401:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:0.0:1790774019.369501:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1790774019.496759:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774019.498810:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774019.560735:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774019.624757:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:2.0:1790774019.688842:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774019.689014:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:2.0:1790774019.689273:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774019.757744:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:0.0:1790774019.816786:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774019.817008:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774019.817046:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:3.0:1790774019.944790:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774019.944900:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:2.0:1790774019.944999:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:1.0:1790774019.945149:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790774020.008800:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774020.008934:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:3.0:1790774020.008991:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774020.009161:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790774020.009263:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790774020.009314:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774020.137107:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774020.202059:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790774020.264784:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774020.328731:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774020.328739:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:3.0:1790774020.329152:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774020.329978:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00010000:00010000:2.0:1790774020.393005:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1790774020.396052:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00000100:00100000:2.0:1790774020.396121:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774020.523401:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:0.0:1790774020.523475:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774020.584851:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774020.649430:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774020.712794:0:14770:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711240 going to sleep for 0 seconds 00000100:00100000:3.0:1790774020.712796:0:13263:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ef2daa9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1790774020.712811:0:15077:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886c00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774020.776969:0:14618:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed9af000 going to sleep for 0 seconds 00000100:00100000:1.0:1790774020.840779:0:15264:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed0c35f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774020.840925:0:14291:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2f00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774020.840937:0:14560:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b26c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774020.968879:0:14361:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a780 going to sleep for 0 seconds 00000100:00100000:3.0:1790774020.969267:0:14478:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee0633600 going to sleep for 0 seconds 00000100:00100000:1.0:1790774020.969355:0:20090:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec5711300 going to sleep for 0 seconds 00000100:00100000:2.0:1790774020.969362:0:14511:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a3c0 going to sleep for 0 seconds 00000100:00100000:3.0:1790774021.032787:0:14869:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774021.032885:0:13563:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca840 going to sleep for 0 seconds 00000100:00100000:0.0:1790774021.033138:0:13565:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774021.033314:0:14884:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec460a240 going to sleep for 0 seconds 00000100:00100000:2.0:1790774021.037092:0:13569:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca6c0 going to sleep for 0 seconds 00000100:00100000:0.0:1790774021.037344:0:13189:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed886300 going to sleep for 0 seconds 00000100:00100000:0.0:1790774021.160749:0:15037:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec8d1e780 going to sleep for 0 seconds 00000100:00100000:2.0:1790774021.224756:0:14538:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed8d53180 going to sleep for 0 seconds 00000100:00100000:1.0:1790774021.288780:0:14184:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eeb1b2e40 going to sleep for 0 seconds 00000100:00100000:0.0:1790774021.352998:0:14589:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eed330b40 going to sleep for 0 seconds 00000100:00100000:1.0:1790774021.353017:0:20095:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ee00b9d80 going to sleep for 0 seconds 00000100:00100000:3.0:1790774021.353327:0:20086:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43cad80 going to sleep for 0 seconds 00000100:00100000:1.0:1790774021.353341:0:15113:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570600 going to sleep for 0 seconds 00000100:00100000:2.0:1790774021.416790:0:20008:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed055af00 going to sleep for 0 seconds 00000100:00100000:1.0:1790774021.416835:0:6156:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec43ca900 going to sleep for 0 seconds 00010000:00010000:2.0:1790774021.416961:0:20143:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1790774021.544867:0:15039:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ed84aa3c0 going to sleep for 0 seconds 00000100:00100000:1.0:1790774021.545021:0:14850:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec7e41a80 going to sleep for 0 seconds 00000100:00100000:2.0:1790774021.608801:0:15310:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4eebbe0f00 going to sleep for 0 seconds 00000100:00100000:0.0:1790774021.672784:0:15031:0:(client.c:2716:ptlrpc_set_wait()) set ffff9c4ec6570f00 going to sleep for 0 seconds Debug log: 97797 lines, 97797 kept, 0 dropped, 0 bad.